TokenProcessor.js 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357
  1. "use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
  2. var _types = require('./parser/tokenizer/types');
  3. var _isAsyncOperation = require('./util/isAsyncOperation'); var _isAsyncOperation2 = _interopRequireDefault(_isAsyncOperation);
  4. class TokenProcessor {
  5. __init() {this.resultCode = ""}
  6. // Array mapping input token index to optional string index position in the
  7. // output code.
  8. __init2() {this.resultMappings = new Array(this.tokens.length)}
  9. __init3() {this.tokenIndex = 0}
  10. constructor(
  11. code,
  12. tokens,
  13. isFlowEnabled,
  14. disableESTransforms,
  15. helperManager,
  16. ) {;this.code = code;this.tokens = tokens;this.isFlowEnabled = isFlowEnabled;this.disableESTransforms = disableESTransforms;this.helperManager = helperManager;TokenProcessor.prototype.__init.call(this);TokenProcessor.prototype.__init2.call(this);TokenProcessor.prototype.__init3.call(this);}
  17. /**
  18. * Snapshot the token state in a way that can be restored later, useful for
  19. * things like lookahead.
  20. *
  21. * resultMappings do not need to be copied since in all use cases, they will
  22. * be overwritten anyway after restore.
  23. */
  24. snapshot() {
  25. return {
  26. resultCode: this.resultCode,
  27. tokenIndex: this.tokenIndex,
  28. };
  29. }
  30. restoreToSnapshot(snapshot) {
  31. this.resultCode = snapshot.resultCode;
  32. this.tokenIndex = snapshot.tokenIndex;
  33. }
  34. /**
  35. * Remove and return the code generated since the snapshot, leaving the
  36. * current token position in-place. Unlike most TokenProcessor operations,
  37. * this operation can result in input/output line number mismatches because
  38. * the removed code may contain newlines, so this operation should be used
  39. * sparingly.
  40. */
  41. dangerouslyGetAndRemoveCodeSinceSnapshot(snapshot) {
  42. const result = this.resultCode.slice(snapshot.resultCode.length);
  43. this.resultCode = snapshot.resultCode;
  44. return result;
  45. }
  46. reset() {
  47. this.resultCode = "";
  48. this.resultMappings = new Array(this.tokens.length);
  49. this.tokenIndex = 0;
  50. }
  51. matchesContextualAtIndex(index, contextualKeyword) {
  52. return (
  53. this.matches1AtIndex(index, _types.TokenType.name) &&
  54. this.tokens[index].contextualKeyword === contextualKeyword
  55. );
  56. }
  57. identifierNameAtIndex(index) {
  58. // TODO: We need to process escapes since technically you can have unicode escapes in variable
  59. // names.
  60. return this.identifierNameForToken(this.tokens[index]);
  61. }
  62. identifierNameAtRelativeIndex(relativeIndex) {
  63. return this.identifierNameForToken(this.tokenAtRelativeIndex(relativeIndex));
  64. }
  65. identifierName() {
  66. return this.identifierNameForToken(this.currentToken());
  67. }
  68. identifierNameForToken(token) {
  69. return this.code.slice(token.start, token.end);
  70. }
  71. rawCodeForToken(token) {
  72. return this.code.slice(token.start, token.end);
  73. }
  74. stringValueAtIndex(index) {
  75. return this.stringValueForToken(this.tokens[index]);
  76. }
  77. stringValue() {
  78. return this.stringValueForToken(this.currentToken());
  79. }
  80. stringValueForToken(token) {
  81. // This is used to identify when two imports are the same and to resolve TypeScript enum keys.
  82. // Ideally we'd process escapes within the strings, but for now we pretty much take the raw
  83. // code.
  84. return this.code.slice(token.start + 1, token.end - 1);
  85. }
  86. matches1AtIndex(index, t1) {
  87. return this.tokens[index].type === t1;
  88. }
  89. matches2AtIndex(index, t1, t2) {
  90. return this.tokens[index].type === t1 && this.tokens[index + 1].type === t2;
  91. }
  92. matches3AtIndex(index, t1, t2, t3) {
  93. return (
  94. this.tokens[index].type === t1 &&
  95. this.tokens[index + 1].type === t2 &&
  96. this.tokens[index + 2].type === t3
  97. );
  98. }
  99. matches1(t1) {
  100. return this.tokens[this.tokenIndex].type === t1;
  101. }
  102. matches2(t1, t2) {
  103. return this.tokens[this.tokenIndex].type === t1 && this.tokens[this.tokenIndex + 1].type === t2;
  104. }
  105. matches3(t1, t2, t3) {
  106. return (
  107. this.tokens[this.tokenIndex].type === t1 &&
  108. this.tokens[this.tokenIndex + 1].type === t2 &&
  109. this.tokens[this.tokenIndex + 2].type === t3
  110. );
  111. }
  112. matches4(t1, t2, t3, t4) {
  113. return (
  114. this.tokens[this.tokenIndex].type === t1 &&
  115. this.tokens[this.tokenIndex + 1].type === t2 &&
  116. this.tokens[this.tokenIndex + 2].type === t3 &&
  117. this.tokens[this.tokenIndex + 3].type === t4
  118. );
  119. }
  120. matches5(t1, t2, t3, t4, t5) {
  121. return (
  122. this.tokens[this.tokenIndex].type === t1 &&
  123. this.tokens[this.tokenIndex + 1].type === t2 &&
  124. this.tokens[this.tokenIndex + 2].type === t3 &&
  125. this.tokens[this.tokenIndex + 3].type === t4 &&
  126. this.tokens[this.tokenIndex + 4].type === t5
  127. );
  128. }
  129. matchesContextual(contextualKeyword) {
  130. return this.matchesContextualAtIndex(this.tokenIndex, contextualKeyword);
  131. }
  132. matchesContextIdAndLabel(type, contextId) {
  133. return this.matches1(type) && this.currentToken().contextId === contextId;
  134. }
  135. previousWhitespaceAndComments() {
  136. let whitespaceAndComments = this.code.slice(
  137. this.tokenIndex > 0 ? this.tokens[this.tokenIndex - 1].end : 0,
  138. this.tokenIndex < this.tokens.length ? this.tokens[this.tokenIndex].start : this.code.length,
  139. );
  140. if (this.isFlowEnabled) {
  141. whitespaceAndComments = whitespaceAndComments.replace(/@flow/g, "");
  142. }
  143. return whitespaceAndComments;
  144. }
  145. replaceToken(newCode) {
  146. this.resultCode += this.previousWhitespaceAndComments();
  147. this.appendTokenPrefix();
  148. this.resultMappings[this.tokenIndex] = this.resultCode.length;
  149. this.resultCode += newCode;
  150. this.appendTokenSuffix();
  151. this.tokenIndex++;
  152. }
  153. replaceTokenTrimmingLeftWhitespace(newCode) {
  154. this.resultCode += this.previousWhitespaceAndComments().replace(/[^\r\n]/g, "");
  155. this.appendTokenPrefix();
  156. this.resultMappings[this.tokenIndex] = this.resultCode.length;
  157. this.resultCode += newCode;
  158. this.appendTokenSuffix();
  159. this.tokenIndex++;
  160. }
  161. removeInitialToken() {
  162. this.replaceToken("");
  163. }
  164. removeToken() {
  165. this.replaceTokenTrimmingLeftWhitespace("");
  166. }
  167. /**
  168. * Remove all code until the next }, accounting for balanced braces.
  169. */
  170. removeBalancedCode() {
  171. let braceDepth = 0;
  172. while (!this.isAtEnd()) {
  173. if (this.matches1(_types.TokenType.braceL)) {
  174. braceDepth++;
  175. } else if (this.matches1(_types.TokenType.braceR)) {
  176. if (braceDepth === 0) {
  177. return;
  178. }
  179. braceDepth--;
  180. }
  181. this.removeToken();
  182. }
  183. }
  184. copyExpectedToken(tokenType) {
  185. if (this.tokens[this.tokenIndex].type !== tokenType) {
  186. throw new Error(`Expected token ${tokenType}`);
  187. }
  188. this.copyToken();
  189. }
  190. copyToken() {
  191. this.resultCode += this.previousWhitespaceAndComments();
  192. this.appendTokenPrefix();
  193. this.resultMappings[this.tokenIndex] = this.resultCode.length;
  194. this.resultCode += this.code.slice(
  195. this.tokens[this.tokenIndex].start,
  196. this.tokens[this.tokenIndex].end,
  197. );
  198. this.appendTokenSuffix();
  199. this.tokenIndex++;
  200. }
  201. copyTokenWithPrefix(prefix) {
  202. this.resultCode += this.previousWhitespaceAndComments();
  203. this.appendTokenPrefix();
  204. this.resultCode += prefix;
  205. this.resultMappings[this.tokenIndex] = this.resultCode.length;
  206. this.resultCode += this.code.slice(
  207. this.tokens[this.tokenIndex].start,
  208. this.tokens[this.tokenIndex].end,
  209. );
  210. this.appendTokenSuffix();
  211. this.tokenIndex++;
  212. }
  213. appendTokenPrefix() {
  214. const token = this.currentToken();
  215. if (token.numNullishCoalesceStarts || token.isOptionalChainStart) {
  216. token.isAsyncOperation = _isAsyncOperation2.default.call(void 0, this);
  217. }
  218. if (this.disableESTransforms) {
  219. return;
  220. }
  221. if (token.numNullishCoalesceStarts) {
  222. for (let i = 0; i < token.numNullishCoalesceStarts; i++) {
  223. if (token.isAsyncOperation) {
  224. this.resultCode += "await ";
  225. this.resultCode += this.helperManager.getHelperName("asyncNullishCoalesce");
  226. } else {
  227. this.resultCode += this.helperManager.getHelperName("nullishCoalesce");
  228. }
  229. this.resultCode += "(";
  230. }
  231. }
  232. if (token.isOptionalChainStart) {
  233. if (token.isAsyncOperation) {
  234. this.resultCode += "await ";
  235. }
  236. if (this.tokenIndex > 0 && this.tokenAtRelativeIndex(-1).type === _types.TokenType._delete) {
  237. if (token.isAsyncOperation) {
  238. this.resultCode += this.helperManager.getHelperName("asyncOptionalChainDelete");
  239. } else {
  240. this.resultCode += this.helperManager.getHelperName("optionalChainDelete");
  241. }
  242. } else if (token.isAsyncOperation) {
  243. this.resultCode += this.helperManager.getHelperName("asyncOptionalChain");
  244. } else {
  245. this.resultCode += this.helperManager.getHelperName("optionalChain");
  246. }
  247. this.resultCode += "([";
  248. }
  249. }
  250. appendTokenSuffix() {
  251. const token = this.currentToken();
  252. if (token.isOptionalChainEnd && !this.disableESTransforms) {
  253. this.resultCode += "])";
  254. }
  255. if (token.numNullishCoalesceEnds && !this.disableESTransforms) {
  256. for (let i = 0; i < token.numNullishCoalesceEnds; i++) {
  257. this.resultCode += "))";
  258. }
  259. }
  260. }
  261. appendCode(code) {
  262. this.resultCode += code;
  263. }
  264. currentToken() {
  265. return this.tokens[this.tokenIndex];
  266. }
  267. currentTokenCode() {
  268. const token = this.currentToken();
  269. return this.code.slice(token.start, token.end);
  270. }
  271. tokenAtRelativeIndex(relativeIndex) {
  272. return this.tokens[this.tokenIndex + relativeIndex];
  273. }
  274. currentIndex() {
  275. return this.tokenIndex;
  276. }
  277. /**
  278. * Move to the next token. Only suitable in preprocessing steps. When
  279. * generating new code, you should use copyToken or removeToken.
  280. */
  281. nextToken() {
  282. if (this.tokenIndex === this.tokens.length) {
  283. throw new Error("Unexpectedly reached end of input.");
  284. }
  285. this.tokenIndex++;
  286. }
  287. previousToken() {
  288. this.tokenIndex--;
  289. }
  290. finish() {
  291. if (this.tokenIndex !== this.tokens.length) {
  292. throw new Error("Tried to finish processing tokens before reaching the end.");
  293. }
  294. this.resultCode += this.previousWhitespaceAndComments();
  295. return {code: this.resultCode, mappings: this.resultMappings};
  296. }
  297. isAtEnd() {
  298. return this.tokenIndex === this.tokens.length;
  299. }
  300. } exports.default = TokenProcessor;