From d3a2df1ea8217dda96f4c7e9579581ac6677fcbe Mon Sep 17 00:00:00 2001 From: c-my Date: Sun, 16 Dec 2018 21:29:03 +0800 Subject: [PATCH] =?UTF-8?q?=E6=A0=B9=E6=8D=AEtokenlist=E7=94=9F=E6=88=90?= =?UTF-8?q?=E5=AD=97=E7=AC=A6=E4=B8=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- res/lexer.c | 2 +- src/com/compiler/Main.java | 11 +- src/com/compiler/lexer/Token.java | 6 +- src/com/compiler/parser/ParserGenerator.java | 212 +++++++++++++++++++ src/com/compiler/parser/ParserTest.java | 5 + src/com/compiler/parser/Symbol.java | 25 ++- src/com/compiler/parser/SymbolString.java | 14 +- src/com/compiler/utils/Show.java | 29 +++ 8 files changed, 296 insertions(+), 8 deletions(-) create mode 100644 src/com/compiler/parser/ParserGenerator.java create mode 100644 src/com/compiler/utils/Show.java diff --git a/res/lexer.c b/res/lexer.c index 5562dfe..4050b47 100644 --- a/res/lexer.c +++ b/res/lexer.c @@ -1,6 +1,6 @@ int main(){ int a=1; - double pi = 3.14; //123 + double pi = 3.14; char[] s = "hello"; char ch = 'c'; } \ No newline at end of file diff --git a/src/com/compiler/Main.java b/src/com/compiler/Main.java index b0ef2a6..59855ec 100644 --- a/src/com/compiler/Main.java +++ b/src/com/compiler/Main.java @@ -2,7 +2,11 @@ package com.compiler; import com.compiler.lexer.Lexer; import com.compiler.lexer.LexerTester; +import com.compiler.parser.ParserGenerator; import com.compiler.parser.ParserTest; +import com.compiler.utils.Show; + +import java.util.ArrayList; public class Main { @@ -12,6 +16,11 @@ public class Main { ParserTest pt = new ParserTest(); Lexer lxer = new Lexer(); LexerTester lt = new LexerTester(); - System.out.println(lt.getTokenList()); + System.out.println(ParserGenerator.ReadTokenlist(lt.getTokenList())); +// ArrayList lines = new ArrayList<>(); +// lines.add("A->tb nB|nC"); +// lines.add("B->tc|ee"); +// Show.printRules(ParserGenerator.Read(lines)); + } } diff --git a/src/com/compiler/lexer/Token.java b/src/com/compiler/lexer/Token.java index 863b6b6..2083beb 100644 --- a/src/com/compiler/lexer/Token.java +++ b/src/com/compiler/lexer/Token.java @@ -3,7 +3,7 @@ package com.compiler.lexer; public class Token { - enum tokenType {INTEGER, FLOAT, DOUBLE, CHAR, STR, IDENTIFIER, DELIMITER, KEYWORD, NONE} + public enum tokenType {INTEGER, FLOAT, DOUBLE, CHAR, STR, IDENTIFIER, DELIMITER, KEYWORD, NONE} private String string = ""; private tokenType type; @@ -21,6 +21,10 @@ public class Token { type = t; } + public tokenType getType() { + return type; + } + public double getDouble() { return Double.parseDouble(string); } diff --git a/src/com/compiler/parser/ParserGenerator.java b/src/com/compiler/parser/ParserGenerator.java new file mode 100644 index 0000000..6fecf63 --- /dev/null +++ b/src/com/compiler/parser/ParserGenerator.java @@ -0,0 +1,212 @@ +package com.compiler.parser; + +import com.compiler.lexer.Token; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +public class ParserGenerator { + + static public HashMap> Read(List lines) { + HashMap> productionRules = new HashMap<>(); + int index = 0; + for (String line : lines) { + var strs = line.split("->"); + if (strs.length < 2) + return new HashMap<>(); + String left = strs[0]; + String right = strs[1]; + + Symbol leftSym = new Symbol(left, Symbol.SymbolType.Nonterminal); + HashSet symbolStrings = new HashSet<>(); + + var symstrs = right.split("\\|"); + for (var str : symstrs) { + SymbolString symbolString = new SymbolString(index++); + var syms = str.split(" "); + for (String sym : syms) { + switch (sym.charAt(0)) { + case 'N': + case 'n': + symbolString.add(new Symbol(sym, Symbol.SymbolType.Nonterminal)); + break; + case 'T': + case 't': + symbolString.add(new Symbol(sym, Symbol.SymbolType.Terminal)); + break; + case 'E': + case 'e': + symbolString.add(new Symbol(sym, Symbol.SymbolType.Empty)); + break; + default: + return new HashMap<>(); + } + } + symbolStrings.add(symbolString); + } + productionRules.put(leftSym, symbolStrings); + } + return productionRules; + } + + static public ArrayList ReadTokenlist(ArrayList tokenList) { + ArrayList symbolList = new ArrayList<>(); + for (Token token : tokenList) { + switch (token.getType()) { + case STR: + symbolList.add(new Symbol("STRING_LITERAL", Symbol.SymbolType.Terminal, token.getStr(), token.getType())); + break; + case IDENTIFIER: + symbolList.add(new Symbol("IDENTIFIER", Symbol.SymbolType.Terminal)); + break; + case DELIMITER: + switch (token.getStr()) { + case "=": + symbolList.add(new Symbol("=", Symbol.SymbolType.Terminal)); + break; + case "*=": + symbolList.add(new Symbol("MUL_ASSIGN", Symbol.SymbolType.Terminal)); + break; + case "/=": + symbolList.add(new Symbol("DIV_ASSIGN", Symbol.SymbolType.Terminal)); + break; + case "%=": + symbolList.add(new Symbol("MOD_ASSIGN", Symbol.SymbolType.Terminal)); + break; + case "+=": + symbolList.add(new Symbol("ADD_ASSIGN", Symbol.SymbolType.Terminal)); + break; + case "-=": + symbolList.add(new Symbol("SUB_ASSIGN", Symbol.SymbolType.Terminal)); + break; + case "<<=": + symbolList.add(new Symbol("LEFT_ASSIGN", Symbol.SymbolType.Terminal)); + break; + case ">>=": + symbolList.add(new Symbol("RIGHT_ASSIGN", Symbol.SymbolType.Terminal)); + break; + case "&=": + symbolList.add(new Symbol("AND_ASSIGN", Symbol.SymbolType.Terminal)); + break; + case "^=": + symbolList.add(new Symbol("XOR_ASSIGN", Symbol.SymbolType.Terminal)); + break; + case "|=": + symbolList.add(new Symbol("OR_ASSIGN", Symbol.SymbolType.Terminal)); + break; + case "++": + symbolList.add(new Symbol("INC_OP", Symbol.SymbolType.Terminal)); + break; + case "--": + symbolList.add(new Symbol("DEC_OP", Symbol.SymbolType.Terminal)); + break; + case "&&": + symbolList.add(new Symbol("AND_OP", Symbol.SymbolType.Terminal)); + break; + case "||": + symbolList.add(new Symbol("OR_OP", Symbol.SymbolType.Terminal)); + break; + case "<=": + symbolList.add(new Symbol("LE_OP", Symbol.SymbolType.Terminal)); + break; + case ">=": + symbolList.add(new Symbol("GE_OP", Symbol.SymbolType.Terminal)); + break; + case "==": + symbolList.add(new Symbol("EQ_OP", Symbol.SymbolType.Terminal)); + break; + case "!=": + symbolList.add(new Symbol("NE_OP", Symbol.SymbolType.Terminal)); + break; + case ";": + symbolList.add(new Symbol(";", Symbol.SymbolType.Terminal)); + break; + case "{": + symbolList.add(new Symbol("{", Symbol.SymbolType.Terminal)); + break; + case "}": + symbolList.add(new Symbol("}", Symbol.SymbolType.Terminal)); + break; + case "(": + symbolList.add(new Symbol("(", Symbol.SymbolType.Terminal)); + break; + case ")": + symbolList.add(new Symbol(")", Symbol.SymbolType.Terminal)); + break; + case "[": + symbolList.add(new Symbol("[", Symbol.SymbolType.Terminal)); + break; + case "]": + symbolList.add(new Symbol("]", Symbol.SymbolType.Terminal)); + break; + case ",": + symbolList.add(new Symbol(",", Symbol.SymbolType.Terminal)); + break; + case ".": + symbolList.add(new Symbol(".", Symbol.SymbolType.Terminal)); + break; + case "&": + symbolList.add(new Symbol("&", Symbol.SymbolType.Terminal)); + break; + case "!": + symbolList.add(new Symbol("!", Symbol.SymbolType.Terminal)); + break; + case "~": + symbolList.add(new Symbol("~", Symbol.SymbolType.Terminal)); + break; + case "->": + symbolList.add(new Symbol("PTR_OP", Symbol.SymbolType.Terminal)); + break; + case "+": + symbolList.add(new Symbol("+", Symbol.SymbolType.Terminal)); + break; + case "-": + symbolList.add(new Symbol("-", Symbol.SymbolType.Terminal)); + break; + case "*": + symbolList.add(new Symbol("*", Symbol.SymbolType.Terminal)); + break; + case "/": + symbolList.add(new Symbol("/", Symbol.SymbolType.Terminal)); + break; + case "%": + symbolList.add(new Symbol("%", Symbol.SymbolType.Terminal)); + break; + case "<": + symbolList.add(new Symbol("<", Symbol.SymbolType.Terminal)); + break; + case ">": + symbolList.add(new Symbol(">", Symbol.SymbolType.Terminal)); + break; + case "^": + symbolList.add(new Symbol("^", Symbol.SymbolType.Terminal)); + break; + case "|": + symbolList.add(new Symbol("|", Symbol.SymbolType.Terminal)); + break; + case "?": + symbolList.add(new Symbol("?", Symbol.SymbolType.Terminal)); + break; + default: + break; + + } + break; + case KEYWORD: + symbolList.add(new Symbol(token.getStr().toUpperCase(), Symbol.SymbolType.Terminal)); + break; + case CHAR: + case INTEGER: + case FLOAT: + case DOUBLE: + symbolList.add(new Symbol("CONSTANT", Symbol.SymbolType.Terminal, + token.getStr(), token.getType())); + break; + + } + } + return symbolList; + } +} diff --git a/src/com/compiler/parser/ParserTest.java b/src/com/compiler/parser/ParserTest.java index 95b1147..4c4a862 100644 --- a/src/com/compiler/parser/ParserTest.java +++ b/src/com/compiler/parser/ParserTest.java @@ -61,4 +61,9 @@ public class ParserTest extends LLParser { return startSym; } + public void initProductionRules(){ +// rules.put(new Symbol("primary_expression", Symbol.SymbolType.Nonterminal),) + + } + } diff --git a/src/com/compiler/parser/Symbol.java b/src/com/compiler/parser/Symbol.java index 72e17df..599b1d7 100644 --- a/src/com/compiler/parser/Symbol.java +++ b/src/com/compiler/parser/Symbol.java @@ -1,5 +1,7 @@ package com.compiler.parser; +import com.compiler.lexer.Token; + public class Symbol { Symbol() { @@ -10,6 +12,14 @@ public class Symbol { Symbol(final String name, SymbolType st) { id = name; sType = st; + value = ""; + } + + Symbol(final String name, SymbolType st, String value, Token.tokenType type) { + id = name; + sType = st; + this.value = value; + this.valueType = type; } enum SymbolType { @@ -24,12 +34,25 @@ public class Symbol { return sType; } + public String getValue() { + return value; + } + private String id; private SymbolType sType; + private String value; + private Token.tokenType valueType = Token.tokenType.NONE; @Override public String toString() { - return id; + StringBuilder sb = new StringBuilder(); + sb.append(id); + if (valueType != Token.tokenType.NONE) { + sb.append("("); + sb.append(value); + sb.append(")"); + } + return sb.toString(); } @Override diff --git a/src/com/compiler/parser/SymbolString.java b/src/com/compiler/parser/SymbolString.java index feb51a6..01d6580 100644 --- a/src/com/compiler/parser/SymbolString.java +++ b/src/com/compiler/parser/SymbolString.java @@ -5,15 +5,20 @@ import java.util.*; public class SymbolString implements Iterable { - SymbolString(List list, int i) { + public SymbolString(int i) { + symString = new ArrayList<>(); + id = i; + } + + public SymbolString(List list, int i) { symString = new ArrayList<>(list); id = i; } /* 创建符号串 - * @param: i 符号串的序号 - * symbols 符号 - */ + * @param: i 符号串的序号 + * symbols 符号 + */ public static SymbolString getSymbolString(int i, String... symbols) { ArrayList array = new ArrayList<>(); for (var sym : symbols) { @@ -30,6 +35,7 @@ public class SymbolString implements Iterable { StringBuilder sb = new StringBuilder(); for (Symbol s : symString) { sb.append(s.toString()); + sb.append(" "); } return sb.toString(); } diff --git a/src/com/compiler/utils/Show.java b/src/com/compiler/utils/Show.java new file mode 100644 index 0000000..a8dd3c8 --- /dev/null +++ b/src/com/compiler/utils/Show.java @@ -0,0 +1,29 @@ +package com.compiler.utils; + +import com.compiler.parser.Symbol; +import com.compiler.parser.SymbolString; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; + +public class Show { + public static void printRules(HashMap> rules){ + for(var rule:rules.keySet()){ + System.out.print(rule); + System.out.print("->"); + var s = rules.get(rule); + for (var symstr:s){ + System.out.print(symstr); + System.out.print("|"); + } + System.out.println(); + } + } + + public static void printSymbolList(ArrayList symList){ + for(Symbol sym:symList){ + System.out.println(sym); + } + } +}