Flex Bison 多线程可重入

背景

Flex Biosn 默认是将状态记录在全局中, 多线程运行时, 会发生混乱。

需要一种多线程编译的技术, 各个线程的 Flex Biosn 独立

环境

OS

1
2
3
4
5
6
7
8
9
10
11
chunli@blog:~$ cat /etc/os-release
PRETTY_NAME="Debian GNU/Linux 11 (bullseye)"
NAME="Debian GNU/Linux"
VERSION_ID="11"
VERSION="11 (bullseye)"
VERSION_CODENAME=bullseye
ID=debian
HOME_URL="https://www.debian.org/"
SUPPORT_URL="https://www.debian.org/support"
BUG_REPORT_URL="https://bugs.debian.org/"
chunli@blog:~$

flex

1
2
3
chunli@blog:~$ flex --version
flex 2.6.4
chunli@blog:~$

bison

1
2
3
4
5
6
7
8
chunli@blog:~$ bison  --version
bison (GNU Bison) 3.7.5
Written by Robert Corbett and Richard Stallman.

Copyright (C) 2021 Free Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
chunli@blog:~$

文件列表

1
2
3
4
5
6
7
8
9
10
11
chunli@blog:~$ ll
total 32K
-rw-r--r-- 1 chunli chunli 11 Mar 15 10:58 calc_in_0.txt
-rw-r--r-- 1 chunli chunli 13 Mar 15 09:58 calc_in_1.txt
-rw-r--r-- 1 chunli chunli 8 Mar 15 09:58 calc_in_2.txt
-rw-r--r-- 1 chunli chunli 12 Mar 15 09:59 calc_in_3.txt
-rw-r--r-- 1 chunli chunli 1.1K Mar 15 10:56 eval.l
-rw-r--r-- 1 chunli chunli 2.2K Mar 15 10:57 eval.y
-rw-r--r-- 1 chunli chunli 1.2K Mar 11 18:12 main.c
-rw-r--r-- 1 chunli chunli 365 Mar 11 17:16 Makefile
chunli@blog:~$

词法分析

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
chunli@blog:~$ cat eval.l
%option noinput nounput noyywrap 8bit nodefault
%option yylineno
%option reentrant bison-bridge bison-locations

%{
#include <stdlib.h>
#include <string.h>
#include "eval.tab.h"

#define YY_USER_ACTION \
yylloc->first_line = yylloc->last_line; \
yylloc->first_column = yylloc->last_column; \
if (yylloc->last_line == yylineno) \
yylloc->last_column += yyleng; \
else { \
yylloc->last_line = yylineno; \
yylloc->last_column = yytext + yyleng - strrchr(yytext, '\n'); \
}
%}

%%

[ \t]+ ;
#.* ;

[[:digit:]]+ printf("Flex File:%s [%s]\n", (const char*)yyget_extra(yyscanner), yytext); *yylval = strtol(yytext, NULL, 0); return NUMBER;

. printf("Flex File:%s [%s]\n", (const char*)yyget_extra(yyscanner), yytext); return *yytext;

\n return *yytext;
chunli@blog:~$

语法分析

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
chunli@blog:~$ cat eval.y
%define api.pure full
%locations
%param { yyscan_t scanner }

%code top {
#include <stdio.h>
}
%code requires {
typedef void* yyscan_t;
}
%code {
#define YY_EXTRA_TYPE void*

int yylex(YYSTYPE* yylvalp, YYLTYPE* yyllocp, yyscan_t scanner);
void yyerror(YYLTYPE* yyllocp, yyscan_t unused, const char* msg);

char *yyget_text ( yyscan_t scanner );
int yyget_leng ( yyscan_t scanner );
FILE *yyget_in ( yyscan_t scanner );
FILE *yyget_out ( yyscan_t scanner );
int yyget_lineno ( yyscan_t scanner );
int yyget_debug ( yyscan_t scanner );
YY_EXTRA_TYPE yyget_extra ( yyscan_t scanner );

void yyset_debug ( int flag, yyscan_t scanner );
void yyset_in ( FILE * in_str , yyscan_t scanner );
void yyset_out ( FILE * out_str , yyscan_t scanner );
void yyset_lineno ( int line_number , yyscan_t scanner );
void yyset_extra ( YY_EXTRA_TYPE user_defined , yyscan_t scanner );
}

%token NUMBER UNOP
%left '+' '-'
%left '*' '/' '%'
%precedence UNOP
%%
input: %empty
| input expr '\n' { fprintf(yyget_out(scanner), "[%d]: %d\n", @2.first_line, $2); }
| input '\n'
| input error '\n' { yyerrok; }
expr : NUMBER
| '(' expr ')' { printf("Bison File:%s 规约识别()\n", (const char*)yyget_extra(scanner)); $$ = $2; }
| '-' expr %prec UNOP { printf("Bison File:%s 规约识别 -\n", (const char*)yyget_extra(scanner)); $$ = -$2; }
| expr '+' expr { printf("Bison File:%s 规约识别 +\n", (const char*)yyget_extra(scanner)); $$ = $1 + $3; }
| expr '-' expr { printf("Bison File:%s 规约识别 -\n", (const char*)yyget_extra(scanner)); $$ = $1 - $3; }
| expr '*' expr { printf("Bison File:%s 规约识别 *\n", (const char*)yyget_extra(scanner)); $$ = $1 * $3; }
| expr '/' expr { printf("Bison File:%s 规约识别 /\n", (const char*)yyget_extra(scanner)); $$ = $1 / $3; }
| expr '%' expr { printf("Bison File:%s 规约识别%%\n", (const char*)yyget_extra(scanner)); $$ = $1 % $3; }

%%

void yyerror(YYLTYPE* yyllocp, yyscan_t scanner, const char* msg)
{
fprintf(stderr, "Bison File:%s [%d:%d]: %s\n", (const char*)yyget_extra(scanner), yyllocp->first_line, yyllocp->first_column, msg);
}

chunli@blog:~$

程序

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
chunli@blog:~$ cat main.c
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include "eval.tab.h"
#include "eval.lex.h"


void* start_routine(void* arg)
{
int index = (size_t)arg;

char filename_in[64];
char filename_out[64];
snprintf(filename_in, sizeof(filename_in), "calc_in_%d.txt", index);
snprintf(filename_out, sizeof(filename_out), "calc_out_%d.txt", index);

void*extra = malloc(64);
memcpy(extra, filename_in, strlen(filename_in)+1);

yyscan_t scanner;
yylex_init(&scanner);
yyset_extra(extra, scanner);
yyset_in(fopen(filename_in, "r"), scanner);
yyset_out(fopen(filename_out, "w+"), scanner);
yyparse(scanner);
yylex_destroy(scanner);

return NULL;
}

#define MAX_THREADS 4
int main(int argc, char* argv[])
{
int i = 0;
pthread_t thread_list[MAX_THREADS];

for(i = 0; i < MAX_THREADS; i++)
{
pthread_create(thread_list + i, NULL, start_routine, (void*)(size_t)i);
}

for(i = 0; i < MAX_THREADS; i++)
{
pthread_join(thread_list[i], NULL);
}

return 0;
}

// https://www.cs.virginia.edu/~cr4bd/flex-manual/Reentrant-Functions.html
chunli@blog:~$

Makefile

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
chunli@blog:~$ cat Makefile
all: eval

eval.lex.c: eval.l
flex -o $@ --header-file=$(patsubst %.c,%.h,$@) --debug $<

eval.tab.c: eval.y
bison -o $@ --defines=$(patsubst %.c,%.h,$@) --debug $<

eval: main.c eval.tab.c eval.lex.c
$(CC) -o $@ -Wall --std=c11 -ggdb -D_XOPEN_SOURCE=700 $(filter %.c,$^) -lpthread

clean:
rm -f eval.tab.c eval.lex.c eval.tab.h eval.lex.h main eval calc_out*

chunli@blog:~$

测试文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
calc_in_0.txt 中 设置了一个错误的语法

chunli@blog:~$ cat calc_in_0.txt
1+77
2*70*
chunli@blog:~$


chunli@blog:~$ cat calc_in_1.txt
3+6*4*5
2+90
chunli@blog:~$


chunli@blog:~$ cat calc_in_2.txt
5*4
5%3
chunli@blog:~$


chunli@blog:~$ cat calc_in_3.txt
66-22
2-8*2
chunli@blog:~$

运行

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
chunli@blog:~$ make clean; make && ./eval

rm -f eval.tab.c eval.lex.c eval.tab.h eval.lex.h main eval calc_out*

bison -o eval.tab.c --defines=eval.tab.h --debug eval.y
flex -o eval.lex.c --header-file=eval.lex.h --debug eval.l
cc -o eval -Wall --std=c11 -ggdb -D_XOPEN_SOURCE=700 main.c eval.tab.c eval.lex.c -lpthread

Flex File:calc_in_3.txt [66]
Flex File:calc_in_3.txt [-]
Flex File:calc_in_3.txt [22]
Bison File:calc_in_3.txt 规约识别 -
Flex File:calc_in_3.txt [2]
Flex File:calc_in_3.txt [-]
Flex File:calc_in_3.txt [8]
Flex File:calc_in_3.txt [*]
Flex File:calc_in_3.txt [2]
Bison File:calc_in_3.txt 规约识别 *
Bison File:calc_in_3.txt 规约识别 -
Flex File:calc_in_1.txt [3]
Flex File:calc_in_1.txt [+]
Flex File:calc_in_1.txt [6]
Flex File:calc_in_1.txt [*]
Flex File:calc_in_1.txt [4]
Bison File:calc_in_1.txt 规约识别 *
Flex File:calc_in_1.txt [*]
Flex File:calc_in_1.txt [5]
Bison File:calc_in_1.txt 规约识别 *
Flex File:calc_in_2.txt [5]
Flex File:calc_in_2.txt [*]
Flex File:calc_in_2.txt [4]
Bison File:calc_in_2.txt 规约识别 *
Flex File:calc_in_2.txt [5]
Flex File:calc_in_2.txt [%]
Flex File:calc_in_2.txt [3]
Bison File:calc_in_2.txt 规约识别%
Flex File:calc_in_0.txt [1]
Flex File:calc_in_0.txt [+]
Flex File:calc_in_0.txt [77]
Bison File:calc_in_0.txt 规约识别 +
Flex File:calc_in_0.txt [2]
Flex File:calc_in_0.txt [*]
Flex File:calc_in_0.txt [70]
Bison File:calc_in_0.txt 规约识别 *
Flex File:calc_in_0.txt [*]
Bison File:calc_in_0.txt [2:6]: syntax error
Bison File:calc_in_1.txt 规约识别 +
Flex File:calc_in_1.txt [2]
Flex File:calc_in_1.txt [+]
Flex File:calc_in_1.txt [90]
Bison File:calc_in_1.txt 规约识别 +
chunli@blog:~$

结果检验

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
chunli@blog:~$ grep '' calc_*
calc_in_0.txt:1+77
calc_in_0.txt:2*70*
calc_in_1.txt:3+6*4*5
calc_in_1.txt:2+90
calc_in_2.txt:5*4
calc_in_2.txt:5%3
calc_in_3.txt:66-22
calc_in_3.txt:2-8*2

calc_out_0.txt:[1]: 78
calc_out_1.txt:[1]: 123
calc_out_1.txt:[2]: 92
calc_out_2.txt:[1]: 20
calc_out_2.txt:[2]: 2
calc_out_3.txt:[1]: 44
calc_out_3.txt:[2]: -14
chunli@blog:~$

准确无误, 基于scanner完全隔离。