--- a/js/src/frontend/BinaryAST.cpp
+++ b/js/src/frontend/BinaryAST.cpp
@@ -16,16 +16,65 @@
#include "jsopcode.h"
#include "SharedContext.h"
namespace js {
namespace frontend {
// #define DEBUG_HEADER 1
+// An abstract data structure on top of string concatenation.
+struct StringTree {
+ struct Leaf {
+ const std::string data;
+ Leaf(const std::string&& data);
+ Leaf(); // Empty leaf
+ };
+ struct Concat {
+ const mozilla::Vector<UniquePtr<StringTree>> children;
+ Concat(mozilla::Vector<UniquePtr<StringTree>>&&);
+ };
+ enum VariantKind {
+ IsLeaf = 128,
+ IsConcat = 129
+ };
+ const VariantKind variantKind;
+ union Variant {
+ Leaf* leaf;
+ Concat* concat;
+ Variant(Leaf* leaf_):
+ leaf(leaf_)
+ {}
+ Variant(Concat* concat_):
+ concat(concat_)
+ {}
+ };
+ const Variant data;
+ const unsigned char kind; // Size-efficient representation of ParseNodeKind.
+
+ // Leaf constructors
+ StringTree(ParseNodeKind, Leaf* leaf);
+ static StringTree* makeLeaf(JSContext* cx, ParseNodeKind);
+ static StringTree* makeLeaf(JSContext* cx, ParseNodeKind, std::string&&);
+ bool isLeaf() const;
+ const std::string& asLeaf() const;
+
+
+ // Concat constructors
+ StringTree(ParseNodeKind, Concat* concat);
+ static StringTree* makeConcat(JSContext* cx, ParseNodeKind, UniquePtr<StringTree>&& singleChild);
+ static StringTree* makeConcat(JSContext* cx, ParseNodeKind, mozilla::Vector<UniquePtr<StringTree>>&& children);
+
+ bool isConcat() const;
+ const Concat& asConcat() const;
+ const mozilla::Vector<UniquePtr<StringTree>>& children() const;
+
+ ~StringTree();
+};
+
using VariantKind = StringTree::VariantKind;
using StringTable = std::unordered_map<std::string, uint32_t>;
using Occurrences = std::pair<std::string, uint32_t>;
bool compareOccurrences(const Occurrences& left, const Occurrences& right) {
return left.second < right.second;
};
@@ -268,18 +317,17 @@ public:
// Now write strings in order.
// To extract a string, you first need to have read the lengths table.
MOZ_ASSERT(heap.size() == lengths.size());
const mozilla::DebugOnly<std::ostringstream::pos_type> startStrings = out.tellp();
for (auto iter: heap) {
out.write(&iter.first[0], iter.first.length());
}
- MOZ_ASSERT((size_t)out.tellp() == (size_t)(startStrings + totalLength));
-
+ MOZ_ASSERT((size_t)out.tellp() == (size_t)startStrings.value + totalLength);
// std::cerr << "treeSerialize: body starts at " << out.tellp() << "\n";
// Finally, write tree
write(tree.get(), out);
return true;
}
@@ -300,17 +348,19 @@ private:
computeOccurrences(tree.get(), occurrences);
}
}
MOZ_MUST_USE StringTree* treeSerializeAux(const ParseNode* node)
{
const ParseNodeKind kind = node ? node->getKind() : PNK_LIMIT;
// std::cerr << "treeSerialize " << NAMES[kind] << "\n";
+#if defined(DEBUG)
debug << NAMES[kind] << " ";
+#endif // defined(DEBUG)
switch (kind) {
// Empty tree
case PNK_LIMIT: {
return StringTree::makeLeaf(cx, PNK_LIMIT);
}
// Body
case PNK_LEXICALSCOPE:
@@ -610,18 +660,20 @@ public:
const std::string sub_str = sub_out.str();
const size_t byteLength = sub_str.length();
writeUnsigned(out, byteLength);
MOZ_ASSERT(byteLength != 0);
const mozilla::DebugOnly<std::ostringstream::pos_type> byteStart = out.tellp();
out << sub_str;
const mozilla::DebugOnly<std::ostringstream::pos_type> byteStop = out.tellp();
- // std::cerr << "StringTree::write length at " << byteStart << "\n";
- MOZ_ASSERT(byteLength == byteStop - byteStart);
+#if defined(DEBUG)
+ // std::cerr << "StringTree::write length at " << byteStart.value << "\n";
+ MOZ_ASSERT(byteLength == (size_t)(byteStop.value - byteStart.value));
+#endif // defined(DEBUG)
}
}
#if DEBUG_HEADER
out.write(FOOTER, sizeof FOOTER - 1);
#endif // DEBUG_HEADER
}
@@ -637,24 +689,28 @@ public:
, functionDepth(0)
, numberOfNodes(0)
, currentByteLength((size_t)-1)
, currentByteStart((size_t)-1)
, currentKind((ParseNodeKind)-1)
, currentVariant((VariantKind)-1)
, allocator(alloc)
, options(options_)
+ , numberOfAllocations(0)
{
// std::cerr << "TreeParser starts with offset " << in.tellg() << "\n";
}
~TreeParser() {
MOZ_ASSERT(in.peek() == std::char_traits<char>::eof());
MOZ_ASSERT(functionDepth == 0);
}
ParseNode* parseNode() {
+
+ const mozilla::TimeStamp start = mozilla::TimeStamp::Now();
+
// Read table of lengths of strings
const uint32_t numberOfStrings = readVariableLength();
if (!lengthsTable.initCapacity(numberOfStrings)) {
MOZ_CRASH();
}
for (uint32_t i = 0; i < numberOfStrings; ++i) {
if (!lengthsTable.append(readVariableLength())) {
MOZ_CRASH();
@@ -671,16 +727,21 @@ public:
in.read(&data[0], iter);
if (!stringsTable.append(Move(data))) {
MOZ_CRASH();
}
}
globalBodyStart = in.tellg();
// std::cerr << "ParseNode: body starts at " << globalBodyStart << "\n";
+
+ const mozilla::TimeStamp stringsStop = mozilla::TimeStamp::Now();
+
+ std::cerr << "ReadBinaryAST strings: " << (stringsStop-start).ToMilliseconds() << "ms\n";
+
return parseNodeAux();
}
private:
// Parse a subtree as a string. The subtree must have been serialized as a string.
// FIXME: Crappy documentation suggests crappy concepts. Clean this up.
uint32_t parseLeaf(ParseNodeKind& kind)
@@ -807,22 +868,27 @@ private:
size_t currentByteStart;
size_t globalBodyStart;
ParseNodeKind currentKind;
VariantKind currentVariant;
mozilla::Vector<uint32_t> lengthsTable;
mozilla::Vector<std::string> stringsTable;
ParseNodeAllocator& allocator;
BinParseOptions options;
+ size_t numberOfAllocations;
JS_DECLARE_NEW_METHODS(new_, allocParseNode, inline)
ParseNode* allocParseNode(size_t size) {
+ numberOfAllocations += 1;
MOZ_ASSERT(size == sizeof(ParseNode));
+ if (MOZ_LIKELY(options.syntaxParse)) {
+ return nullptr;
+ }
return static_cast<ParseNode*>(allocator.allocNode());
}
void handleHeader()
{
++numberOfNodes;
#if DEBUG_HEADER
// std::cerr << "handleHeader HEADER at " << in.tellg() << "\n";
@@ -835,16 +901,17 @@ private:
// std::cerr << "handleHeader kind at " << in.tellg() << "\n";
unsigned char char_kind;
in.read((char*)&char_kind, sizeof char_kind);
if (char_kind >= mozilla::ArrayLength(NAMES)) {
MOZ_CRASH("Bad kind");
}
currentKind = (ParseNodeKind)char_kind;
+ // std::cerr << "handleHeader: " << NAMES[char_kind] << "\n";
if (currentKind == PNK_LIMIT) {
currentByteLength = 0;
currentVariant = VariantKind::IsLeaf;
} else {
#if DEBUG_HEADER
// std::cerr << "handleHeader variantKind at " << in.tellg() << "\n";
unsigned char char_variant;
@@ -855,21 +922,23 @@ private:
#endif // DEBUG_HEADER
}
}
void handleFooter() {
// std::cerr << "handleFooter " << in.tellg() << "\n";
// Check byteLength, footer
+#if defined(DEBUG)
MOZ_ASSERT(currentVariant == VariantKind::IsConcat || currentVariant == VariantKind::IsLeaf);
if (currentVariant == VariantKind::IsConcat) {
- const mozilla::DebugOnly<std::istringstream::pos_type> byteStop = in.tellg();
+ const size_t byteStop = in.tellg();
MOZ_ASSERT(byteStop - currentByteStart == currentByteLength);
}
+#endif // defined(DEBUG)
#if DEBUG_HEADER
std::string footer(sizeof FOOTER - 1, '\0');
in.read(&footer[0], sizeof FOOTER - 1);
if (footer.compare(0, sizeof FOOTER - 1, FOOTER) != 0) {
MOZ_CRASH("Bad footer");
}
#endif // DEBUG_HEADER
@@ -898,61 +967,63 @@ private:
void getLeafAsString(uint32_t id, std::string& data)
{
data = stringsTable[id];
}
MOZ_MUST_USE ParseNode* parseNode(ParseNodeKind kind)
{
+#if defined(DEBUG)
debug << NAMES[kind] << " ";
+#endif // defined(DEBUG)
switch (kind) {
// Empty subtree.
case PNK_LIMIT: {
return nullptr;
}
case PNK_LEXICALSCOPE: {
const mozilla::DebugOnly<size_t> length = readNodeAsConcat();
MOZ_ASSERT(length == 1);
UniquePtr<ParseNode> body(parseNodeAux());
- MOZ_ASSERT(body);
return new_<LexicalScopeNode>(nullptr, body.release());
}
case PNK_FUNCTION: {
// Allocate a value to simulate the cost of really storing the
// data that would let us parse lazily.
struct BogusFunctionData {
size_t codeStart;
size_t byteLength;
};
- if (functionDepth == options.lazyParseAfterFunctionDepth) {
- UniquePtr<CodeNode> result(new_<CodeNode>(kind, JSOP_NOP, TokenPos()));
+ if (functionDepth == options.skipContentsAfterFunctionDepth) {
+ UniquePtr<CodeNode> result(new_<CodeNode>(kind, JSOP_NOP, TokenPos(0, 0)));
UniquePtr<BogusFunctionData> position(new BogusFunctionData());
position->codeStart = in.tellg();
skipConcatNode();
position->byteLength = currentByteLength;
return result.release();
}
++functionDepth;
const mozilla::DebugOnly<size_t> length = readNodeAsConcat();
MOZ_ASSERT(length == 1);
UniquePtr<ParseNode> body(parseNodeAux());
- MOZ_ASSERT(body);
- CodeNode* result(new_<CodeNode>(kind, JSOP_NOP, TokenPos()));
- result->pn_body = body.release();
+ CodeNode* result(new_<CodeNode>(kind, JSOP_NOP, TokenPos(0, 0)));
+ if (result) {
+ result->pn_body = body.release();
+ }
// result->pn_funbox = box;
--functionDepth;
return result;
}
case PNK_IF: MOZ_FALLTHROUGH;
case PNK_IMPORT_SPEC: MOZ_FALLTHROUGH;
case PNK_EXPORT: MOZ_FALLTHROUGH;
@@ -962,17 +1033,17 @@ private:
case PNK_TRY: MOZ_FALLTHROUGH;
case PNK_CATCH: {
const mozilla::DebugOnly<size_t> length = readNodeAsConcat();
MOZ_ASSERT(length == 3);
UniquePtr<ParseNode> kid1(parseNodeAux());
UniquePtr<ParseNode> kid2(parseNodeAux());
UniquePtr<ParseNode> kid3(parseNodeAux());
- return new_<TernaryNode>(kind, JSOP_NOP, kid1.release(), kid2.release(), kid3.release(), TokenPos());
+ return new_<TernaryNode>(kind, JSOP_NOP, kid1.release(), kid2.release(), kid3.release(), TokenPos(0, 0));
}
case PNK_CASE: MOZ_FALLTHROUGH;
case PNK_WHILE: MOZ_FALLTHROUGH;
case PNK_DOWHILE: MOZ_FALLTHROUGH;
case PNK_FOR: MOZ_FALLTHROUGH;
case PNK_COMPREHENSIONFOR: MOZ_FALLTHROUGH;
case PNK_WITH: MOZ_FALLTHROUGH;
@@ -1008,17 +1079,17 @@ private:
UniquePtr<ParseNode> left(parseNodeAux());
UniquePtr<ParseNode> right(parseNodeAux());
if (kind == PNK_CASE) {
return new_<CaseClause>(left.release(), right.release(), 0);
} else if (kind == PNK_FORIN || kind == PNK_FOROF) {
return new_<TernaryNode>(kind, JSOP_NOP, left.release(), nullptr, right.release());
} else {
- return new_<BinaryNode>(kind, JSOP_NOP, TokenPos(), left.release(), right.release());
+ return new_<BinaryNode>(kind, JSOP_NOP, TokenPos(0, 0), left.release(), right.release());
}
}
// pn_kid
case PNK_THROW: MOZ_FALLTHROUGH;
case PNK_EXPORT_DEFAULT: MOZ_FALLTHROUGH;
case PNK_COMPUTED_NAME: MOZ_FALLTHROUGH;
case PNK_ARRAYPUSH: MOZ_FALLTHROUGH;
@@ -1045,17 +1116,17 @@ private:
case PNK_DELETENAME: MOZ_FALLTHROUGH;
case PNK_DELETEPROP: MOZ_FALLTHROUGH;
case PNK_DELETEELEM: MOZ_FALLTHROUGH;
case PNK_DELETEEXPR: MOZ_FALLTHROUGH;
case PNK_MUTATEPROTO: {
const mozilla::DebugOnly<size_t> length = readNodeAsConcat();
MOZ_ASSERT(length == 1);
- return new_<UnaryNode>(kind, JSOP_NOP, TokenPos(), parseNodeAux());
+ return new_<UnaryNode>(kind, JSOP_NOP, TokenPos(0, 0), parseNodeAux());
}
// Linked lists
case PNK_CLASSMETHODLIST: MOZ_FALLTHROUGH;
case PNK_EXPORT_SPEC_LIST: MOZ_FALLTHROUGH;
case PNK_STATEMENTLIST: MOZ_FALLTHROUGH;
case PNK_CATCHLIST: MOZ_FALLTHROUGH;
case PNK_VAR: MOZ_FALLTHROUGH;
@@ -1094,49 +1165,51 @@ private:
case PNK_TAGGED_TEMPLATE: MOZ_FALLTHROUGH;
case PNK_CALLSITEOBJ: MOZ_FALLTHROUGH;
case PNK_IMPORT_SPEC_LIST: MOZ_FALLTHROUGH;
case PNK_SUPERCALL: MOZ_FALLTHROUGH;
case PNK_ARRAYCOMP: MOZ_FALLTHROUGH;
case PNK_INSTANCEOF: MOZ_FALLTHROUGH;
case PNK_IN: {
const size_t length = readNodeAsConcat();
- ParseNode* latest = nullptr;
- UniquePtr<ListNode> result(new_<ListNode>(kind, TokenPos()));
+ UniquePtr<ListNode> result(new_<ListNode>(kind, TokenPos(0, 0)));
for (uint32_t i = 0; i < length; ++i) {
ParseNode* current = parseNodeAux();
if (i == 0) {
- result->pn_head = current;
- result->pn_tail = &result->pn_head;
+ if (result) {
+ result->initList(current);
+ }
} else {
- latest->pn_next = current;
- result->pn_tail = &latest->pn_next;
+ if (result) {
+ result->append(current);
+ }
}
- latest = current;
- result->pn_count++; // Incrementing progressively in case it helps if the destructor is called early.
}
+#if defined(DEBUG)
+ result->checkListConsistency();
+#endif // defined(DEBUG)
return result.release();
}
// Labels
case PNK_BREAK: MOZ_FALLTHROUGH;
case PNK_CONTINUE: {
std::string leaf;
getLeafAsString(readNodeAsLeaf(), leaf);
RootedPropertyName label(cx);
if (!deserializePropertyName(cx, leaf, &label)) {
label = nullptr;
}
if (kind == PNK_BREAK) {
- return new_<BreakStatement>(label, TokenPos());
+ return new_<BreakStatement>(label, TokenPos(0, 0));
} else {
- return new_<ContinueStatement>(label, TokenPos());
+ return new_<ContinueStatement>(label, TokenPos(0, 0));
}
}
case PNK_LABEL: MOZ_FALLTHROUGH;
case PNK_DOT: {
const mozilla::DebugOnly<size_t> length = readNodeAsConcat();
MOZ_ASSERT(length == 2);
@@ -1161,82 +1234,86 @@ private:
// Strings
case PNK_NAME: MOZ_FALLTHROUGH;
case PNK_STRING: MOZ_FALLTHROUGH;
case PNK_TEMPLATE_STRING: {
std::string leaf;
getLeafAsString(readNodeAsLeaf(), leaf);
RootedAtom atom(cx, deserializeAtom(cx, leaf));
if (kind == PNK_NAME) {
- return new_<NameNode>(kind, JSOP_NOP, atom, TokenPos());
+ return new_<NameNode>(kind, JSOP_NOP, atom, TokenPos(0, 0));
} else {
- return new_<NullaryNode>(kind, JSOP_NOP, TokenPos(), atom);
+ return new_<NullaryNode>(kind, JSOP_NOP, TokenPos(0, 0), atom);
}
}
case PNK_NUMBER: {
std::string leaf;
getLeafAsString(readNodeAsLeaf(), leaf);
std::istringstream input(leaf); // FIXME: Useless copy.
double dval;
input.read((char*)&dval, sizeof dval);
DecimalPoint point;
input.read((char*)&point, sizeof point);
- UniquePtr<ParseNode> node(new_<ParseNode>(kind, JSOP_NOP, PN_NULLARY));
- node->initNumber(dval, point);
+ UniquePtr<ParseNode> node(new_<ParseNode>(kind, JSOP_NOP, PN_NULLARY, TokenPos(0, 0)));
+ if (node) {
+ node->initNumber(dval, point);
+ }
return node.release();
}
// Nullaries
case PNK_TRUE: MOZ_FALLTHROUGH;
case PNK_FALSE: MOZ_FALLTHROUGH;
case PNK_NULL: MOZ_FALLTHROUGH;
case PNK_RAW_UNDEFINED: MOZ_FALLTHROUGH;
case PNK_GENERATOR: MOZ_FALLTHROUGH;
case PNK_NOP: MOZ_FALLTHROUGH;
case PNK_EXPORT_BATCH_SPEC: MOZ_FALLTHROUGH;
case PNK_POSHOLDER: {
mozilla::Unused << readNodeAsLeaf();
- return new_<NullaryNode>(kind, JSOP_NOP, TokenPos());
+ return new_<NullaryNode>(kind, JSOP_NOP, TokenPos(0, 0));
}
// Entirely undocumented nodes:
case PNK_MODULE: MOZ_FALLTHROUGH;
case PNK_DEBUGGER: MOZ_FALLTHROUGH;
case PNK_ELISION: MOZ_FALLTHROUGH;
case PNK_OBJECT_PROPERTY_NAME: {
mozilla::Unused << readNodeAsLeaf();
- return new_<NullaryNode>(kind, JSOP_NOP, TokenPos());
+ return new_<NullaryNode>(kind, JSOP_NOP, TokenPos(0, 0));
}
// Stuff we don't handle yet:
case PNK_REGEXP: {
std::string leaf;
getLeafAsString(readNodeAsLeaf(), leaf);
- return new_<RegExpLiteral>(nullptr, TokenPos());
+ return new_<RegExpLiteral>(nullptr, TokenPos(0, 0));
}
}
MOZ_CRASH("treeParse: out of switch()");
return nullptr;
}
};
MOZ_MUST_USE bool binSerialize(JSContext* cx, const ParseNode* node, std::ostringstream& out, std::ostringstream& debug) {
TreeSerializer serializer(cx, debug);
return serializer.treeSerialize(node, out);
}
-MOZ_MUST_USE ParseNode* binParse(JSContext* cx, ParseNodeAllocator& alloc, std::istringstream& in,
- std::ostringstream& debug, BinParseOptions& options
+MOZ_MUST_USE bool binParse(JSContext* cx, ParseNodeAllocator& alloc, std::istringstream& in,
+ std::ostringstream& debug, BinParseOptions& options,
+ ParseNode** result
) {
TreeParser parser(cx, alloc, in, debug, options);
- return parser.parseNode();
+ *result = parser.parseNode();
+ return true;
}
} // namespace frontend
} // namespace js
--- a/js/src/shell/js.cpp
+++ b/js/src/shell/js.cpp
@@ -4432,25 +4432,32 @@ ReadBinaryAST(JSContext* cx, unsigned ar
JS_ReportErrorNumberASCII(cx, my_GetErrorMessage, nullptr, JSSMSG_INVALID_ARGS,
"evaluate");
return false;
}
RootedObject opts(cx, &args[1].toObject());
RootedValue val(cx);
- if (!JS_GetProperty(cx, opts, "lazyParseAfterFunctionDepth", &val)) {
+ if (!JS_GetProperty(cx, opts, "skipContentsAfterFunctionDepth", &val)) {
return false;
}
if (!val.isUndefined()) {
double out;
if (!ToNumber(cx, val, &out)) {
return false;
}
- options.lazyParseAfterFunctionDepth = out;
+ options.skipContentsAfterFunctionDepth = out;
+ }
+
+ if (!JS_GetProperty(cx, opts, "syntaxParse", &val)) {
+ return false;
+ }
+ if (!val.isUndefined()) {
+ options.syntaxParse = ToBoolean(val);
}
}
RootedObject obj(cx, &args[0].toObject());
if (!JS_IsArrayBufferObject(obj)) {
const char* typeName = InformalValueTypeName(args[0]);
JS_ReportErrorASCII(cx, "expected typed array to parse, got %s", typeName);
return false;
@@ -4467,21 +4474,31 @@ ReadBinaryAST(JSContext* cx, unsigned ar
// In a real implementation, we'll probably want to replace istringstream with something better.
std::cerr << "ReadBinaryAST from " << buf_length << " bytes at " << (void*)buf_data << "\n";
std::cerr << "ReadBinaryAST string: " << buf_str.length() << "\n";
std::ostringstream _debug; // Ignored.
// in_stream.exceptions(in_stream.exceptions() | std::istringstream::badbit | std::istringstream::failbit | std::istringstream::eofbit );
ParseNodeAllocator allocator(cx, cx->tempLifoAlloc());
+
+ // Force GC, to kill cache and reset allocator.
+ my_LargeAllocFailCallback();
+
const mozilla::TimeStamp start = mozilla::TimeStamp::Now();
- if (!binParse(cx, allocator, in_stream, _debug, options)) {
+ ParseNode* node = nullptr;
+ if (!binParse(cx, allocator, in_stream, _debug, options, &node)) {
return false;
}
const mozilla::TimeStamp stop = mozilla::TimeStamp::Now();
+ MOZ_ASSERT(!!node == !options.syntaxParse);
+ if (node) {
+ allocator.freeTree(node);
+ }
+
std::cerr << "ReadBinaryAST duration: " << (stop-start).ToMilliseconds() << "ms\n";
args.rval().setUndefined();
return true;
}
static bool
ParseAux(JSContext* cx, unsigned argc, Value* vp, ParseOptions& instructions)
{
@@ -6659,17 +6676,17 @@ static const JSFunctionSpecWithHelp shel
JS_FN_HELP("jsbinSerialize", CreateBinaryAST, 1, 0,
"jsbinSerialize(source)",
" Compile a source string into a binary AST.\n"),
JS_FN_HELP("jsbinParse", ReadBinaryAST, 1, 0,
"jsbinParse(typedarray, [options])",
" Read an AST from a binary AST\n"
-" lazyParseAfterFunctionDepth: (number).\n"),
+" skipContentsAfterFunctionDepth: (number).\n"),
JS_FN_HELP("offThreadCompileScript", OffThreadCompileScript, 1, 0,
"offThreadCompileScript(code[, options])",
" Compile |code| on a helper thread. To wait for the compilation to finish\n"
" and run the code, call |runOffThreadScript|. If present, |options| may\n"
" have properties saying how the code should be compiled:\n"
" noScriptRval: use the no-script-rval compiler option (default: false)\n"
" fileName: filename for error messages and debug info\n"