Skip to content

Commit 41f4de8

Browse files
author
Nikos M
committed
v.3.1.0
* (positive/negative) lookahead tokens implementations * various changes
1 parent c4b6a04 commit 41f4de8

File tree

5 files changed

+46
-42
lines changed

5 files changed

+46
-42
lines changed

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ see [Modularity and Future Directions](https://github.com/foo123/editor-grammar/
4242
* [`Grammar.Syntax Model`](https://github.com/foo123/editor-grammar/blob/master/grammar-reference.md#syntax-model) can enable highlight in a more *context-specific* way, plus detect possible *syntax errors*
4343
* [`Grammar.Syntax Model`](https://github.com/foo123/editor-grammar/blob/master/grammar-reference.md#syntax-model) can contain **recursive references**
4444
* [`Grammar.Syntax Model`](https://github.com/foo123/editor-grammar/blob/master/grammar-reference.md#syntax-pegbnf-like-notations) can be (fully) specificed using [`PEG`](https://en.wikipedia.org/wiki/Parsing_expression_grammar)-like notation or [`BNF`](https://en.wikipedia.org/wiki/Backus%E2%80%93Naur_Form)-like notation (**NEW feature**)
45+
* [`Grammar.Syntax Model`](https://github.com/foo123/editor-grammar/blob/master/grammar-reference.md#syntax-pegbnf-like-notations) implements **positive / negative lookahead tokens** (analogous to `PEG` `and-`/`not-` entities) (**NEW feature**)
4546
* `Grammar` can define [*action* tokens](https://github.com/foo123/editor-grammar/blob/master/grammar-reference.md#action-tokens) to perform *complex context-specific* parsing functionality, including **associated tag matching** and **duplicate identifiers** (see for example `xml.grammar` example) (**NEW feature**)
4647
* Generated highlighters are **optimized for speed and size**
4748
* Can generate a syntax-highlighter from a grammar **interactively and on-the-fly** ( see example, http://foo123.github.io/examples/prism-grammar )

‎beeld.config‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ tasks =[{}]
4949

5050
"@@ROOT@@" = "this"
5151
"@@EXPORTS@@" = "exports"
52-
"@@VERSION@@" = "3.0.0"
52+
"@@VERSION@@" = "3.1.0"
5353
"@@MODULE_NAME@@" = "PrismGrammar"
5454

5555
@

‎build/prism_grammar.js‎

Lines changed: 40 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/**
22
*
33
* PrismGrammar
4-
* @version: 3.0.0
4+
* @version: 3.1.0
55
*
66
* Transform a grammar specification in JSON format, into a syntax-highlighter for Prism
77
* https://github.com/foo123/prism-grammar
@@ -26,7 +26,7 @@ else if ( !(name in root) ) /* Browser/WebWorker/.. */
2626
"use strict";
2727
/**
2828
* EditorGrammar Codebase
29-
* @version: 3.0.0
29+
* @version: 3.1.0
3030
*
3131
* https://github.com/foo123/editor-grammar
3232
**/
@@ -60,7 +60,7 @@ T_BLOCK = 1024, T_COMMENT = 1025,
6060
T_ALTERNATION = 2048,
6161
T_SEQUENCE = 4096,
6262
T_REPEATED = 8192, T_ZEROORONE = 8193, T_ZEROORMORE = 8194, T_ONEORMORE = 8195,
63-
T_LOOKAHEAD = 16384, T_POSITIVE_LOOKAHEAD = T_LOOKAHEAD, T_NEGATIVE_LOOKAHEAD = 16385, /*TODO*/
63+
T_LOOKAHEAD = 16384, T_POSITIVE_LOOKAHEAD = T_LOOKAHEAD, T_NEGATIVE_LOOKAHEAD = 16385,
6464
T_NGRAM = 32768,
6565
T_SEQUENCE_OR_NGRAM = T_SEQUENCE|T_NGRAM,
6666
T_COMPOSITE = T_ALTERNATION|T_SEQUENCE|T_REPEATED|T_LOOKAHEAD|T_NGRAM,
@@ -1528,20 +1528,19 @@ function parse_peg_bnf_notation( tok, Lex, Syntax )
15281528

15291529
else if ( '&' === c || '!' === c )
15301530
{
1531-
// TODO
15321531
// lookahead modifier, applies to token that comes before
1533-
/*if ( sequence.length )
1532+
if ( sequence.length )
15341533
{
15351534
prev_token = sequence[sequence.length-1];
15361535
curr_token = '' + prev_token + c;
15371536
if ( !Syntax[ curr_token ] )
15381537
Syntax[ curr_token ] = {
1539-
type:'&' === c ? 'positiveLookahead' : 'negativeLookahead',
1538+
type:'!' === c ? 'negativeLookahead' : 'positiveLookahead',
15401539
tokens:[prev_token]
15411540
}
15421541
sequence[sequence.length-1] = curr_token;
15431542
}
1544-
else*/ token += c;
1543+
else token += c;
15451544
}
15461545

15471546
else if ( '|' === c )
@@ -1858,7 +1857,7 @@ function get_tokenizer( tokenID, RegExpID, Lex, Syntax, Style,
18581857
{
18591858
autocompletions = get_autocomplete(
18601859
token,
1861-
make_array( T_STR_OR_ARRAY & get_type(token.autocomplete) ? token.autocomplete : token.tokens ),
1860+
T_STR_OR_ARRAY&get_type(token.autocomplete) ? make_array( token.autocomplete ) : $tokens$,
18621861
tokenID,
18631862
keywords
18641863
);
@@ -1941,7 +1940,6 @@ function get_tokenizer( tokenID, RegExpID, Lex, Syntax, Style,
19411940
{
19421941
if ( T_POSITIVE_LOOKAHEAD === $type$ || T_NEGATIVE_LOOKAHEAD === $type$ )
19431942
{
1944-
// TODO
19451943
$token$ = new tokenizer( $type$, tokenID, null, $msg$, $modifier$ );
19461944
}
19471945
else if ( (T_REPEATED & $type$) && (T_ARRAY & get_type( token.repeat )) )
@@ -2877,8 +2875,13 @@ function t_composite( t, stream, state, token )
28772875
do {
28782876
tokenizer = t_clone( tokens[ i0++ ], is_sequence, modifier, $id );
28792877
style = tokenize( tokenizer, stream, state, token );
2880-
// bypass failed but optional tokens in the sequence and get to the next ones
2881-
} while (/*is_sequence &&*/ i0 < n && false === style && !(tokenizer.status & REQUIRED/*_OR_ERROR*/));
2878+
// bypass failed but optional tokens in the sequence
2879+
// or successful lookahead tokens
2880+
// and get to the next ones
2881+
} while (/*is_sequence &&*/ i0 < n && (
2882+
((true === style) && (T_LOOKAHEAD & tokenizer.type)) ||
2883+
((false === style) && !(tokenizer.status & REQUIRED/*_OR_ERROR*/))
2884+
));
28822885

28832886
if ( false !== style )
28842887
{
@@ -2907,16 +2910,13 @@ function t_composite( t, stream, state, token )
29072910
return false;
29082911
}
29092912

2910-
else if ( T_POSITIVE_LOOKAHEAD === type )
2911-
{
2912-
// TODO
2913-
self.status = 0; return false;
2914-
}
2915-
2916-
else if ( T_NEGATIVE_LOOKAHEAD === type )
2913+
else if ( T_LOOKAHEAD & type )
29172914
{
2918-
// TODO
2919-
self.status = 0; return false;
2915+
tokenizer = t_clone( tokens[ 0 ], 1, modifier, $id );
2916+
style = tokenize( tokenizer, stream, state, token );
2917+
if ( stream.pos > stream_pos ) stream.bck( stream_pos );
2918+
if ( stack.length > stack_pos ) stack.length = stack_pos;
2919+
return T_NEGATIVE_LOOKAHEAD === type ? false === style : false !== style;
29202920
}
29212921

29222922
else //if ( T_REPEATED & type )
@@ -3250,8 +3250,9 @@ var Parser = Class({
32503250
// error
32513251
if ( tokenizer.status & REQUIRED_OR_ERROR )
32523252
{
3253-
// empty the stack of the syntax rule group of this tokenizer
3253+
// keep it for autocompletion, if needed
32543254
state.token = tokenizer;
3255+
// empty the stack of the syntax rule group of this tokenizer
32553256
empty( stack, tokenizer.$id /*|| true*/ );
32563257
// skip this
32573258
if ( !stream.nxt( true ) ) { stream.spc( ); just_space = true; }
@@ -3269,18 +3270,8 @@ var Parser = Class({
32693270
// found token
32703271
else
32713272
{
3272-
// action token(s) follow, execute action(s) on current token
3273-
if ( stack.length && T_ACTION === stack[stack.length-1].type )
3274-
{
3275-
while ( stack.length && T_ACTION === stack[stack.length-1].type )
3276-
{
3277-
action = stack.pop(); t_action( action, stream, state, token );
3278-
// action error
3279-
if ( action.status & ERROR ) state.$actionerr$ = true;
3280-
}
3281-
}
32823273
// partial block, apply maybe any action(s) following it
3283-
else if ( stack.length > 1 && stream.eol() &&
3274+
if ( stack.length > 1 && stream.eol() &&
32843275
(T_BLOCK & stack[stack.length-1].type) && state.block &&
32853276
state.block.name === stack[stack.length-1].name
32863277
)
@@ -3293,6 +3284,17 @@ var Parser = Class({
32933284
if ( action.status & ERROR ) state.$actionerr$ = true;
32943285
}
32953286
}
3287+
// action token(s) follow, execute action(s) on current token
3288+
else if ( stack.length && (T_ACTION === stack[stack.length-1].type) )
3289+
{
3290+
while ( stack.length && (T_ACTION === stack[stack.length-1].type) )
3291+
{
3292+
action = stack.pop();
3293+
t_action( action, stream, state, token );
3294+
// action error
3295+
if ( action.status & ERROR ) state.$actionerr$ = true;
3296+
}
3297+
}
32963298
// not empty
32973299
if ( true !== type ) { notfound = false; break; }
32983300
}
@@ -3328,21 +3330,22 @@ var Parser = Class({
33283330
return T;
33293331
}
33303332

3331-
,autocompletion: function( state ) {
3333+
,autocompletion: function( state, min_found ) {
33323334
var stack = state.stack, i, token, type,
33333335
hash = {}, follows = generate_autocompletion( [ state.token ], [], hash );
3336+
min_found = min_found || 0;
33343337
for(i=stack.length-1; i>=0; i--)
33353338
{
33363339
token = stack[ i ]; type = token.type;
33373340
if ( T_REPEATED & type )
33383341
{
33393342
follows = generate_autocompletion( [ token ], follows, hash );
3340-
if ( (0 < token.min) && follows.length ) break;
3343+
if ( (0 < token.min) && (min_found < follows.length) ) break;
33413344
}
33423345
else if ( (T_SIMPLE === type) || (T_ALTERNATION === type) || (T_SEQUENCE_OR_NGRAM & type) )
33433346
{
33443347
follows = generate_autocompletion( [ token ], follows, hash );
3345-
if ( follows.length ) break;
3348+
if ( min_found < follows.length ) break;
33463349
}
33473350
}
33483351
return follows;
@@ -3416,7 +3419,7 @@ var Parser = Class({
34163419
/**
34173420
*
34183421
* PrismGrammar
3419-
* @version: 3.0.0
3422+
* @version: 3.1.0
34203423
*
34213424
* Transform a grammar specification in JSON format, into a syntax-highlighter for Prism
34223425
* https://github.com/foo123/prism-grammar
@@ -3593,7 +3596,7 @@ function get_mode( grammar, Prism )
35933596
[/DOC_MARKDOWN]**/
35943597
var PrismGrammar = exports['PrismGrammar'] = {
35953598

3596-
VERSION: "3.0.0",
3599+
VERSION: "3.1.0",
35973600

35983601
// clone a grammar
35993602
/**[DOC_MARKDOWN]

‎build/prism_grammar.min.js‎

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎editor-grammar‎

0 commit comments

Comments
 (0)