/
common.py
525 lines (476 loc) · 23.2 KB
/
common.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
#!/bin/false
# This file is part of Espruino, a JavaScript interpreter for Microcontrollers
#
# Copyright (C) 2013 Gordon Williams <gw@pur3.co.uk>
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# ----------------------------------------------------------------------------------------
# Reads board information from boards/BOARDNAME.py - used by build_board_docs,
# build_pininfo, and build_platform_config
# ----------------------------------------------------------------------------------------
# Global
import subprocess;
import re;
import json;
import sys;
import os;
import importlib;
import traceback;
# Local
import pinutils;
# Exported - this is set if a board is specified on the command-line
board = False
silent = os.getenv("SILENT");
if silent:
class Discarder(object):
def write(self, text):
pass # do nothing
def flush(self):
pass # do nothing
# now discard everything coming out of stdout
sys.stdout = Discarder()
# http://stackoverflow.com/questions/4814970/subprocess-check-output-doesnt-seem-to-exist-python-2-6-5
if "check_output" not in dir( subprocess ):
def f(*popenargs, **kwargs):
if 'stdout' in kwargs:
raise ValueError('stdout argument not allowed, it will be overridden.')
process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
output, unused_err = process.communicate()
retcode = process.poll()
if retcode:
cmd = kwargs.get("args")
if cmd is None:
cmd = popenargs[0]
raise subprocess.CalledProcessError(retcode, cmd)
return output
subprocess.check_output = f
#
# Scans files for comments of the form /*JSON......*/
#
# Comments look like:
#
#/*JSON{ "type":"staticmethod|staticproperty|constructor|method|property|function|variable|class|library|idle|init|kill|EV_xxx",
# // class = built-in class that does not require instantiation
# // library = built-in class that needs require('classname')
# // idle = function to run on idle regardless
# // hwinit = function to run on Hardware Initialisation (called once at boot time, after jshInit, before jsvInit/etc)
# // init = function to run on Initialisation (eg boot/load/reset/after save/etc)
# // kill = function to run on Deinitialisation (eg before save/reset/etc)
# // EV_xxx = Something to be called with a character in an IRQ when it is received (eg. EV_SERIAL1)
# "class" : "Double", "name" : "doubleToIntBits",
# "needs_parentName":true, // optional - if for a method, this makes the first 2 args parent+parentName (not just parent)
# "generate_full|generate|wrap" : "*(JsVarInt*)&x", // if generate=false, it'll only be used for docs
# "generate_js" : "full/file/path.js", // you can supply a JS file instead of 'generate' above. Should be of the form '(function(args) { ... })'
# "description" : " Convert the floating point value given into an integer representing the bits contained in it",
# "params" : [ [ "x" , "float|int|int32|bool|pin|JsVar|JsVarName|JsVarArray", "A floating point number"] ],
# // float - parses into a JsVarFloat which is passed to the function
# // int - parses into a JsVarInt which is passed to the function
# // int32 - parses into a 32 bit int
# // bool - parses into a boolean
# // pin - parses into a pin
# // JsVar - passes a JsVar* to the function (after skipping names)
# // JsVarArray - parses this AND ANY SUBSEQUENT ARGUMENTS into a JsVar of type JSV_ARRAY. THIS IS ALWAYS DEFINED, EVEN IF ZERO LENGTH. Currently it must be the only parameter
# "return" : ["int|float|JsVar", "The integer representation of x"],
# "return_object" : "ObjectName", // optional - used for tern's code analysis - so for example we can do hints for openFile(...).yyy
# "no_create_links":1 // optional - if this is set then hyperlinks are not created when this name is mentioned (good example = bit() )
# "no_docs":1 // optional - if this is set then documentation is not created for this entry
# "not_real_object" : "anything", // optional - for classes, this means we shouldn't treat this as a built-in object, as internally it isn't stored in a JSV_OBJECT
# "prototype" : "Object", // optional - for classes, this is what their prototype is. It's particlarly helpful if not_real_object, because there is no prototype var in that case
# "check" : "jsvIsFoo(var)", // for classes - this is code that returns true if 'var' is of the given type
# "ifndef" : "SAVE_ON_FLASH", // if the given preprocessor macro is defined, don't implement this
# "ifdef" : "USE_LCD_FOO", // if the given preprocessor macro isn't defined, don't implement this
# "#if" : "A>2", // add a #if statement in the generated C file (ONLY if type==object)
# "patch" : true, // if true, this isn't a complete JSON, but just updates another with the same class+name
# "sortorder" : 0 // default to 0, but all items are sorted by this first, so especially with jswrap_X_init/etc we can ensure the ordering is correct
#}*/
#
# description can be an array of strings as well as a simple string (in which case each element is separated by a newline),
# and adding ```sometext``` in the description surrounds it with HTML code tags
#
# COMMAND LINE OPTIONS
# -Ddefinition
# -BBOARDFILE
#
# Also adds 'defines' field to global 'board' object (if boardObject is defined)
def get_jsondata(is_for_document, parseArgs = True, boardObject = False):
global board # use the board object defined above
board = boardObject
scriptdir = os.path.dirname (os.path.realpath(__file__))
print("Script location "+scriptdir)
os.chdir(scriptdir+"/..")
ignore_ifdefs = is_for_document
# C files that we'll scan for JSON data
jswraps = []
# definitions that are used when evaluating IFDEFs/etc
defines = []
explicit_files = False
if parseArgs and len(sys.argv)>1:
print("Using files from command line")
for i in range(1,len(sys.argv)):
arg = sys.argv[i]
if arg[0]=="-":
if arg[1]=="D":
defines.append(arg[2:])
if "=" in arg: # eg ESPR_EMBED=1 needs to also have a define for ESPR_EMBED
defines.append(arg[2:arg.index("=")])
elif arg[1]=="B":
print("BOARD "+arg[2:]);
print("Now ignore_ifdefs = False");
ignore_ifdefs = False
board = importlib.import_module(arg[2:])
elif arg[1]=="F":
"" # -Fxxx.yy in args is filename xxx.yy, which is mandatory for build_jswrapper.py
else:
print("Unknown command-line option")
exit(1)
elif arg[-2:]==".c":
# C file, all good
explicit_files = True
jswraps.append(arg)
else:
print("WARNING: Ignoring unknown file type: " + arg)
if not explicit_files:
print("Scanning for jswrap.c files")
jswraps = subprocess.check_output(["find", ".", "-name", "jswrap*.c"]).strip().decode("utf-8").split("\n")
if board:
board.defines = defines
if "usart" in board.chip: defines.append("ESPR_USART_COUNT="+str(board.chip["usart"]));
if "spi" in board.chip: defines.append("ESPR_SPI_COUNT="+str(board.chip["spi"]));
if "i2c" in board.chip: defines.append("ESPR_I2C_COUNT="+str(board.chip["i2c"]));
if "USB" in board.devices: defines.append("defined(USB)=True");
else: defines.append("defined(USB)=False");
if "build" in board.info:
if "defines" in board.info["build"]:
for i in board.info["build"]["defines"]:
print("board.defines: " + i);
defines.append(i)
if "makefile" in board.info["build"]:
for i in board.info["build"]["makefile"]:
print("board.makefile: " + i);
i = i.strip()
if i.startswith("DEFINES"):
defs = i[7:].strip()[2:].strip().split() # array of -Dsomething
for d in defs:
if not d.startswith("-D"):
print("WARNING: expecting -Ddefine, got " + d)
defines.append(d[2:])
if len(defines)>1:
print("Got #DEFINES:")
for d in defines: print(" "+d)
githash = get_git_hash()
if len(githash)==0: githash="master"
jsondatas = []
for jswrap in jswraps:
# ignore anything from archives
if jswrap.startswith("./archives/"): continue
# now scan
print("Scanning "+jswrap)
code = open(jswrap, "r").read()
if is_for_document and not explicit_files and "DO_NOT_INCLUDE_IN_DOCS" in code:
print("FOUND 'DO_NOT_INCLUDE_IN_DOCS' IN FILE "+jswrap)
continue
for comment in re.findall(r"/\*JSON.*?\*/", code, re.VERBOSE | re.MULTILINE | re.DOTALL):
charnumber = code.find(comment)
linenumber = 1+code.count("\n", 0, charnumber)
# Strip off /*JSON .. */ bit
comment = comment[6:-2]
endOfJson = comment.find("\n}")+2;
jsonstring = comment[0:endOfJson];
description = comment[endOfJson:].strip();
# print("Parsing "+jsonstring)
try:
jsondata = json.loads(jsonstring)
if len(description): jsondata["description"] = description;
jsondata["filename"] = jswrap
if jswrap[-2:]==".c":
jsondata["include"] = jswrap[:-2]+".h"
jsondata["githublink"] = "https://github.com/espruino/Espruino/blob/"+githash+"/"+jswrap+"#L"+str(linenumber)
dropped_prefix = "Dropped "
if "name" in jsondata: dropped_prefix += jsondata["name"]+" "
elif "class" in jsondata: dropped_prefix += jsondata["class"]+" "
drop = False
if is_for_document and ("no_docs" in jsondata):
print(dropped_prefix+" because of 'no_docs' tag")
drop = True
if not ignore_ifdefs:
if ("generate" in jsondata) and jsondata["generate"]==False and not is_for_document:
print(dropped_prefix+" because of generate=false")
drop = True
if ("ifndef" in jsondata) and (jsondata["ifndef"] in defines):
print(dropped_prefix+" because of #ifndef "+jsondata["ifndef"])
drop = True
if ("ifdef" in jsondata) and not (jsondata["ifdef"] in defines):
print(dropped_prefix+" because of #ifdef "+jsondata["ifdef"])
drop = True
if ("#ifdef" in jsondata) or ("#ifndef" in jsondata):
sys.stderr.write( "'#ifdef' where 'ifdef' should be used in " + jsonstring + " - "+str(sys.exc_info()[0]) + "\n" )
exit(1)
if ("if" in jsondata):
sys.stderr.write( "'if' where '#if' should be used in " + jsonstring + " - "+str(sys.exc_info()[0]) + "\n" )
exit(1)
if ("#if" in jsondata):
expr = jsondata["#if"]
for defn in defines:
expr = expr.replace("defined("+defn+")", "True");
if defn.find('=')!=-1:
dname = defn[:defn.find('=')]
dkey = defn[defn.find('=')+1:]
expr = expr.replace("defined("+dname+")", "True");
expr = expr.replace(dname, dkey);
# Now replace any defined(...) we haven't heard of with false
expr = re.sub(r"defined\([^\)]*\)", "False", expr)
expr = expr.replace("||","or").replace("&&","and");
expr = expr.replace("!","not ");
try:
r = eval(expr)
except:
print("WARNING: error evaluating '"+expr+"' - from '"+jsondata["#if"]+"'")
r = True
if not r:
print(dropped_prefix+" because of #if "+jsondata["#if"]+ " -> "+expr)
drop = True
if not drop and "patch" in jsondata:
targetjsondata = [x for x in jsondatas if x["type"]==jsondata["type"] and x["class"]==jsondata["class"] and x["name"]==jsondata["name"]]
if len(targetjsondata) > 0:
targetjsondata = targetjsondata[0]
for key in jsondata:
if not key in ["type","class","name","patch"]:
print("Copying "+key+" --- "+jsondata[key])
targetjsondata[key] = jsondata[key]
drop = True
if not drop:
jsondatas.append(jsondata)
except ValueError as e:
sys.stderr.write( "JSON PARSE FAILED for " + jsonstring + " - "+ str(e) + "\n")
exc_obj = sys.exc_info()
print(''.join(traceback.format_exception(exc_obj)))
exit(1)
except Exception as e:
sys.stderr.write( "JSON PARSE FAILED for " + jsonstring + " - "+str(e) + "\n" )
exc_obj = sys.exc_info()
print(''.join(traceback.format_exception(exc_obj)))
exit(1)
print("Scanning finished.")
if board:
for device in pinutils.SIMPLE_DEVICES:
if device in board.devices and not "novariable" in board.devices[device]:
jsondatas.append({
"type" : "variable",
"name" : device,
"generate_full" : device+"_PININDEX",
"return" : ["pin", device],
"filename" : "BOARD.py",
"include" : "platform_config.h"
})
if "LED1" in board.devices and not "novariable" in board.devices["LED1"]:
jsondatas.append({
"type" : "variable",
"name" : "LED",
"generate_full" : "LED1_PININDEX",
"return" : ["pin", "LED1"],
"filename" : "BOARD.py",
"include" : "platform_config.h"
})
if "BTN1" in board.devices and not "novariable" in board.devices["BTN1"]:
jsondatas.append({
"type" : "variable",
"name" : "BTN",
"generate_full" : "BTN1_PININDEX",
"return" : ["pin", "Button 1"],
"filename" : "BOARD.py",
"include" : "platform_config.h"
})
jsondatas = sorted(jsondatas, key=lambda j: j["sortorder"] if "sortorder" in j else 0)
return jsondatas
# Takes the data from get_jsondata and restructures it in prepartion for output as JS
#
# Results look like:,
#{
# "Pin": {
# "desc": [
# "This is the built-in class for Pins, such as D0,D1,LED1, or BTN",
# "You can call the methods on Pin, or you can use Wiring-style functions such as digitalWrite"
# ],
# "methods": {
# "read": {
# "desc": "Returns the input state of the pin as a boolean",
# "params": [],
# "return": [
# "bool",
# "Whether pin is a logical 1 or 0"
# ]
# },
# "reset": {
# "desc": "Sets the output state of the pin to a 0",
# "params": [],
# "return": []
# },
# ...
# },
# "props": {},
# "staticmethods": {},
# "staticprops": {}
# },
# "print": {
# "desc": "Print the supplied string",
# "return": []
# },
# ...
#}
#
def get_struct_from_jsondata(jsondata):
context = {"modules": {}}
def checkClass(details):
cl = details["class"]
if not cl in context:
context[cl] = {"type": "class", "methods": {}, "props": {}, "staticmethods": {}, "staticprops": {}, "desc": details.get("description", "")}
return cl
def addConstructor(details):
cl = checkClass(details)
context[cl]["constructor"] = {"params": details.get("params", []), "return": details.get("return", []), "desc": details.get("description", "")}
def addMethod(details, type = ""):
cl = checkClass(details)
context[cl][type + "methods"][details["name"]] = {"params": details.get("params", []), "return": details.get("return", []), "desc": details.get("description", "")}
def addProp(details, type = ""):
cl = checkClass(details)
context[cl][type + "props"][details["name"]] = {"return": details.get("return", []), "desc": details.get("description", "")}
def addFunc(details):
context[details["name"]] = {"type": "function", "return": details.get("return", []), "desc": details.get("description", "")}
def addObj(details):
context[details["name"]] = {"type": "object", "instanceof": details.get("instanceof", ""), "desc": details.get("description", "")}
def addLib(details):
context["modules"][details["class"]] = {"desc": details.get("description", "")}
def addVar(details):
return
for data in jsondata:
type = data["type"]
if type=="class":
checkClass(data)
elif type=="constructor":
addConstructor(data)
elif type=="method":
addMethod(data)
elif type=="property":
addProp(data)
elif type=="staticmethod":
addMethod(data, "static")
elif type=="staticproperty":
addProp(data, "static")
elif type=="function":
addFunc(data)
elif type=="object":
addObj(data)
elif type=="library":
addLib(data)
elif type=="variable":
addVar(data)
else:
print(json.dumps(data, sort_keys=True, indent=2))
return context
def get_includes_from_jsondata(jsondatas):
includes = []
for jsondata in jsondatas:
if "include" in jsondata:
include = jsondata["include"]
if not include in includes:
includes.append(include)
return includes
def is_property(jsondata):
return jsondata["type"]=="property" or jsondata["type"]=="staticproperty" or jsondata["type"]=="variable"
def is_function(jsondata):
return jsondata["type"]=="function" or jsondata["type"]=="method"
def get_prefix_name(jsondata):
if jsondata["type"]=="event": return "event"
if jsondata["type"]=="constructor": return "constructor"
if jsondata["type"]=="function": return "function"
if jsondata["type"]=="method": return "function"
if jsondata["type"]=="variable": return "variable"
if jsondata["type"]=="property": return "property"
return ""
def get_ifdef_description(d):
if d=="SAVE_ON_FLASH": return "devices with low flash memory"
if d=="SAVE_ON_FLASH_EXTREME": return "devices with extremely low flash memory (eg. HYSTM32_28)"
if d=="STM32": return "STM32 devices (including Espruino Original, Pico and WiFi)"
if d=="STM32F1": return "STM32F1 devices (including Original Espruino Board)"
if d=="NRF52_SERIES": return "NRF52 devices (like Puck.js, Pixl.js, Bangle.js and MDBT42Q)"
if d=="PUCKJS": return "Puck.js devices"
if d=="PIXLJS": return "Pixl.js boards"
if d=="ESPRUINOWIFI": return "Espruino WiFi boards"
if d=="ESPRUINOBOARD": return "'Original' Espruino boards"
if d=="PICO": return "Espruino Pico boards"
if d=="BANGLEJS": return "Bangle.js smartwatches"
if d=="BANGLEJS_F18": return "Bangle.js 1 smartwatches"
if d=="BANGLEJS_Q3" or d=="BANGLEJS2": return "Bangle.js 2 smartwatches"
if d=="ESPR_EMBED": return "Embeddable Espruino C builds"
if d=="ESP8266": return "ESP8266 boards running Espruino"
if d=="ESP32": return "ESP32 boards"
if d=="EFM32": return "EFM32 devices"
if d=="MICROBIT": return "BBC micro:bit boards"
if d=="MICROBIT2": return "BBC micro:bit v2 boards"
if d=="USE_LCD_SDL": return "Linux with SDL support compiled in"
if d=="USE_TLS": return "devices with TLS and SSL support (Espruino Pico and Espruino WiFi only)"
if d=="RELEASE": return "release builds"
if d=="DEBUG": return "debug builds"
if d=="LINUX": return "Linux-based builds"
if d=="BLUETOOTH": return "devices with Bluetooth LE capability"
if d=="USB": return "devices with USB"
if d=="USE_USB_HID": return "devices that support USB HID (Espruino Pico and Espruino WiFi)"
if d=="USE_AES": return "devices that support AES (Espruino Pico, Espruino WiFi or Linux)"
if d=="USE_SHA256": return "devices that support SHA256 (Espruino Pico, Espruino WiFi, Espruino BLE devices or Linux)"
if d=="USE_SHA512": return "devices that support SHA512 (Espruino Pico, Espruino WiFi, Espruino BLE devices or Linux)"
if d=="USE_CRYPTO": return "devices that support Crypto Functionality (Espruino Pico, Original, Espruino WiFi, Espruino BLE devices, Linux or ESP8266)"
if d=="USE_FLASHFS": return "devices with filesystem in Flash support enabled (ESP32 only)"
if d=="USE_TERMINAL": return "devices with VT100 terminal emulation enabled (Pixl.js only)"
if d=="USE_TELNET": return "devices with Telnet enabled (Linux, ESP8266 and ESP32)"
if d=="USE_WIZNET": return "builds with support for WIZnet Ethernet modules built in"
if d=="USE_NFC": return "NFC (Puck.js, Pixl.js, MDBT42Q)"
if d=="GRAPHICS_ANTIALIAS": return "devices with Antialiasing support included (Bangle.js or Linux)"
print("WARNING: Unknown ifdef '"+d+"' in common.get_ifdef_description")
return d
def get_script_dir():
return os.path.dirname(os.path.realpath(__file__))
def get_git_hash():
return subprocess.check_output('git log -1 --format="%h"', shell=True).strip().decode("utf-8")
def get_version():
# Warning: the same release label derivation is also in the Makefile
scriptdir = get_script_dir()
jsutils = scriptdir+"/../src/jsutils.h"
version = re.compile("^.*JS_VERSION.*\"(.*)\"");
alt_release = os.getenv("ALT_RELEASE")
if alt_release == None:
# Default release labeling based on commits since last release tag
latest_release = subprocess.check_output('git tag | grep RELEASE_ | sort | tail -1', shell=True).strip()
commits_since_release = subprocess.check_output('git log --oneline '+latest_release.decode("utf-8")+'..HEAD | wc -l', shell=True).decode("utf-8").strip()
else:
# Alternate release labeling with fork name (in ALT_RELEASE env var) plus branch
# name plus commit SHA
sha = subprocess.check_output('git rev-parse --short HEAD', shell=True).strip()
branch = subprocess.check_output('git name-rev --name-only HEAD', shell=True).strip()
commits_since_release = alt_release + '_' + branch + '_' + sha
for line in open(jsutils):
match = version.search(line);
if (match != None):
v = match.group(1);
if commits_since_release=="0": return v
else: return v+"."+commits_since_release
return "UNKNOWN"
def get_name_or_space(jsondata):
if "name" in jsondata: return jsondata["name"]
return ""
def get_bootloader_size(board):
if board.chip["family"]=="STM32F4": return 16*1024; # 16kb Pages, so we have no choice
return 10*1024;
# On normal chips this is 0x00000000
# On boards with bootloaders it's generally + 10240
# On F401, because of the setup of pages we put the bootloader in the first 16k, then in the 16+16+16 we put the saved code, and then finally we but the binary somewhere else
def get_espruino_binary_address(board):
if "place_text_section" in board.chip:
return board.chip["place_text_section"]
if "bootloader" in board.info and board.info["bootloader"]==1:
return get_bootloader_size(board);
return 0;
def get_board_binary_name(board):
return board.info["binary_name"].replace("%v", get_version());