[开发测试]前端实现简单的词法分析(生成tokens)

const LETTERS = /[a-z0-9]/;
const LeftParentheses = "LeftParentheses";
const RightParentheses = "RightParentheses";
const JSXIdentifier = "JSXIdentifier";
const AttributeKey = "AttributeKey";
const JSXText = "JSXText";
const AttributeStringValue = "AttributeStringValue";
const BackSlash = "BackSlash";

let currentToken = { type: "", value: "" };
const tokens = [];

function emit(token) {
  tokens.push(token);
  currentToken = { type: "", value: "" };
}

function start(char) {
  if (char === "<") {
    emit({ type: LeftParentheses, value: char });
    return foundLeftParentheses; //找到了<
  }
  throw new Error("第一个字符必须是<");
}

function eof() {
  if (currentToken.value.length > 0) {
    emit(currentToken);
  }
}

function foundLeftParentheses(char) {
  if (LETTERS.test(char)) {
    currentToken.type = JSXIdentifier;
    currentToken.value += char;
    return jsxIdentifier;
  } else if (char === "/") {
    emit({ type: BackSlash, value: char });
    currentToken.type = JSXIdentifier;
    return jsxIdentifier;
  }
}

function foundRightParentheses(char) {
  if (char === "<") {
    emit({ type: LeftParentheses, value: char });
    return foundLeftParentheses;
  } else {
    currentToken.type = JSXText;
    currentToken.value += char;
    return jsxText;
  }
}

function jsxText(char) {
  if (char === "<") {
    emit(currentToken);
    emit({ type: LeftParentheses, value: char });
    return foundLeftParentheses;
  } else {
    currentToken.value += char;
    return jsxText;
  }
}

function jsxIdentifier(char) {
  if (LETTERS.test(char)) {
    currentToken.value += char;
    return jsxIdentifier;
  } else if (char === " ") {
    emit(currentToken);
    return attribute;
  } else if (char === ">") {
    emit(currentToken);
    emit({ type: RightParentheses, value: char });
    return foundRightParentheses;
  }
  return eof;
}

function attribute(char) {
  if (LETTERS.test(char)) {
    currentToken.type = AttributeKey;
    currentToken.value += char;
    return attributeKey;
  }
  throw new TypeError("Error");
}

function attributeKey(char) {
  if (LETTERS.test(char)) {
    currentToken.value += char;
    return attributeKey;
  } else if (char === "=") {
    emit(currentToken);
    return attributeValue;
  }
  throw new TypeError("Error");
}

function attributeValue(char) {
  if (char === '"') {
    currentToken.type = AttributeStringValue;
    return attributeStringValue;
  }
}

function attributeStringValue(char) {
  if (LETTERS.test(char)) {
    currentToken.value += char;
    return attributeStringValue;
  } else if (char === '"') {
    emit(currentToken);
    return tryLeaveAttribute;
  }
}

function tryLeaveAttribute(char) {
  if (char === " ") {
    return attribute;
  } else if (char === ">") {
    emit({ type: RightParentheses, value: char });
    return foundRightParentheses;
  }
}

function tokenizer(input) {
  let state = start;
  for (let char of input) {
    if (state) {
      state = state(char);
    }
  }
  return tokens;
}

module.exports = {
  tokenizer,
};

// let sourceCode = '<h1 id="title" class="title"><span>Hello</span>world</h1>';
let sourceCode =
  '<h1 id="title" class="title"><span id="title">Hello</span>world</h1>';

console.log(tokenizer(sourceCode));
/* [
  { type: 'LeftParentheses', value: '<' },
  { type: 'JSXIdentifier', value: 'h1' },
  { type: 'AttributeKey', value: 'id' },
  { type: 'AttributeStringValue', value: 'title' },
  { type: 'AttributeKey', value: 'class' },
  { type: 'AttributeStringValue', value: 'title' },
  { type: 'RightParentheses', value: '>' },
  { type: 'LeftParentheses', value: '<' },
  { type: 'JSXIdentifier', value: 'span' },
  { type: 'AttributeKey', value: 'id' },
  { type: 'AttributeStringValue', value: 'title' },
  { type: 'RightParentheses', value: '>' },
  { type: 'JSXText', value: 'Hello' },
  { type: 'LeftParentheses', value: '<' },
  { type: 'BackSlash', value: '/' },
  { type: 'JSXIdentifier', value: 'span' },
  { type: 'RightParentheses', value: '>' },
  { type: 'JSXText', value: 'world' },
  { type: 'LeftParentheses', value: '<' },
  { type: 'BackSlash', value: '/' },
  { type: 'JSXIdentifier', value: 'h1' },
  { type: 'RightParentheses', value: '>' }
]*/