1 /* 2 * Copyright (C) 2023 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef SRC_TRACE_PROCESSOR_PERFETTO_SQL_TOKENIZER_SQLITE_TOKENIZER_H_ 18 #define SRC_TRACE_PROCESSOR_PERFETTO_SQL_TOKENIZER_SQLITE_TOKENIZER_H_ 19 20 #include <cstdint> 21 #include <string_view> 22 #include <utility> 23 24 #include "src/trace_processor/perfetto_sql/grammar/perfettosql_grammar.h" 25 #include "src/trace_processor/sqlite/sql_source.h" 26 27 namespace perfetto::trace_processor { 28 29 // Tokenizes SQL statements according to SQLite SQL language specification: 30 // https://www2.sqlite.org/hlr40000.html 31 // 32 // Usage of this class: 33 // SqliteTokenizer tzr(std::move(my_sql_source)); 34 // for (auto t = tzr.Next(); t.token_type != TK_SEMI; t = tzr.Next()) { 35 // // Handle t here 36 // } 37 class SqliteTokenizer { 38 public: 39 // A single SQL token according to the SQLite standard. 40 struct Token { 41 // The string contents of the token. 42 std::string_view str; 43 44 // The type of the token. 45 int token_type = TK_ILLEGAL; 46 47 bool operator==(const Token& o) const { 48 return str == o.str && token_type == o.token_type; 49 } 50 51 // Returns if the token is empty or semicolon. IsTerminalToken52 bool IsTerminal() const { return token_type == TK_SEMI || str.empty(); } 53 }; 54 55 enum class EndToken { 56 kExclusive, 57 kInclusive, 58 }; 59 60 // Creates a tokenizer which tokenizes |sql|. 61 explicit SqliteTokenizer(SqlSource sql); 62 63 SqliteTokenizer(const SqliteTokenizer&) = delete; 64 SqliteTokenizer& operator=(const SqliteTokenizer&) = delete; 65 66 SqliteTokenizer(SqliteTokenizer&&) = delete; 67 SqliteTokenizer& operator=(SqliteTokenizer&&) = delete; 68 69 // Returns the next SQL token. 70 Token Next(); 71 72 // Returns the next SQL token which is not of type TK_SPACE. 73 Token NextNonWhitespace(); 74 75 // Returns the next SQL token which is terminal. 76 Token NextTerminal(); 77 78 // Returns an SqlSource containing all the tokens between |start| and |end|. 79 // 80 // Note: |start| and |end| must both have been previously returned by this 81 // tokenizer. If |end_token| == kInclusive, the end token is also included 82 // in the substring. 83 SqlSource Substr(const Token& start, 84 const Token& end, 85 EndToken end_token = EndToken::kExclusive) const; 86 87 // Returns an SqlSource containing only the SQL backing |token|. 88 // 89 // Note: |token| must have been previously returned by this tokenizer. 90 SqlSource SubstrToken(const Token& token) const; 91 92 // Returns a traceback error message for the SqlSource backing this tokenizer 93 // pointing to |token|. See SqlSource::AsTraceback for more information about 94 // this method. 95 // 96 // Note: |token| must have been previously returned by this tokenizer. 97 std::string AsTraceback(const Token&) const; 98 99 // Replaces the SQL in |rewriter| between |start| and |end| with the contents 100 // of |rewrite|. If |end_token| == kInclusive, the end token is also included 101 // in the rewrite. 102 void Rewrite(SqlSource::Rewriter& rewriter, 103 const Token& start, 104 const Token& end, 105 SqlSource rewrite, 106 EndToken end_token = EndToken::kExclusive) const; 107 108 // Replaces the SQL in |rewriter| backing |token| with the contents of 109 // |rewrite|. 110 void RewriteToken(SqlSource::Rewriter&, 111 const Token&, 112 SqlSource rewrite) const; 113 114 // Resets this tokenizer to tokenize |source|. Any previous returned tokens 115 // are invalidated. Reset(SqlSource source)116 void Reset(SqlSource source) { 117 source_ = std::move(source); 118 offset_ = 0; 119 } 120 121 private: 122 SqlSource source_; 123 uint32_t offset_ = 0; 124 }; 125 126 } // namespace perfetto::trace_processor 127 128 #endif // SRC_TRACE_PROCESSOR_PERFETTO_SQL_TOKENIZER_SQLITE_TOKENIZER_H_ 129