1 /*
2 * Copyright (C) 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/sqlite/sqlite_engine.h"
18
19 #include <sqlite3.h>
20 #include <cstdint>
21 #include <optional>
22 #include <string>
23 #include <utility>
24
25 #include "perfetto/base/build_config.h"
26 #include "perfetto/base/logging.h"
27 #include "perfetto/base/status.h"
28 #include "perfetto/public/compiler.h"
29 #include "src/trace_processor/sqlite/scoped_db.h"
30 #include "src/trace_processor/sqlite/sql_source.h"
31 #include "src/trace_processor/tp_metatrace.h"
32
33 #include "protos/perfetto/trace_processor/metatrace_categories.pbzero.h"
34
35 // In Android and Chromium tree builds, we don't have the percentile module.
36 // Just don't include it.
37 #if PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
38 // defined in sqlite_src/ext/misc/percentile.c
39 extern "C" int sqlite3_percentile_init(sqlite3* db,
40 char** error,
41 const sqlite3_api_routines* api);
42 #endif // PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
43
44 namespace perfetto::trace_processor {
45 namespace {
46
EnsureSqliteInitialized()47 void EnsureSqliteInitialized() {
48 // sqlite3_initialize isn't actually thread-safe in standalone builds because
49 // we build with SQLITE_THREADSAFE=0. Ensure it's only called from a single
50 // thread.
51 static bool init_once = [] {
52 // Enabling memstatus causes a lock to be taken on every malloc/free in
53 // SQLite to update the memory statistics. This can cause massive contention
54 // in trace processor when multiple instances are used in parallel.
55 // Fix this by disabling the memstatus API which we don't make use of in
56 // any case. See b/335019324 for more info on this.
57 int ret = sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0);
58
59 // As much as it is painful, we need to catch instances of SQLITE_MISUSE
60 // here against all the advice of the SQLite developers and lalitm@'s
61 // intuition: SQLITE_MISUSE for sqlite3_config really means: that someone
62 // else has already initialized SQLite. As we are an embeddable library,
63 // it's very possible that the process embedding us has initialized SQLite
64 // in a different way to what we want to do and, if so, we should respect
65 // their choice.
66 //
67 // TODO(lalitm): ideally we would have an sqlite3_is_initialized API we
68 // could use to gate the above check but that doesn't exist: report this
69 // issue to SQLite developers and see if such an API could be added. If so
70 // we can remove this check.
71 if (ret == SQLITE_MISUSE) {
72 return true;
73 }
74
75 PERFETTO_CHECK(ret == SQLITE_OK);
76 return sqlite3_initialize() == SQLITE_OK;
77 }();
78 PERFETTO_CHECK(init_once);
79 }
80
InitializeSqlite(sqlite3 * db)81 void InitializeSqlite(sqlite3* db) {
82 char* error = nullptr;
83 sqlite3_exec(db, "PRAGMA temp_store=2", nullptr, nullptr, &error);
84 if (error) {
85 PERFETTO_FATAL("Error setting pragma temp_store: %s", error);
86 }
87 // In Android tree builds, we don't have the percentile module.
88 #if PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
89 sqlite3_percentile_init(db, &error, nullptr);
90 if (error) {
91 PERFETTO_ELOG("Error initializing: %s", error);
92 sqlite3_free(error);
93 }
94 #endif
95 }
96
GetErrorOffsetDb(sqlite3 * db)97 std::optional<uint32_t> GetErrorOffsetDb(sqlite3* db) {
98 int offset = sqlite3_error_offset(db);
99 return offset == -1 ? std::nullopt
100 : std::make_optional(static_cast<uint32_t>(offset));
101 }
102
103 } // namespace
104
SqliteEngine()105 SqliteEngine::SqliteEngine() {
106 sqlite3* db = nullptr;
107 EnsureSqliteInitialized();
108
109 // Ensure that we open the database with mutexes disabled: this is because
110 // trace processor as a whole cannot be used from multiple threads so there is
111 // no point paying the (potentially significant) cost of mutexes at the SQLite
112 // level.
113 static constexpr int kSqliteOpenFlags =
114 SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX;
115 PERFETTO_CHECK(sqlite3_open_v2(":memory:", &db, kSqliteOpenFlags, nullptr) ==
116 SQLITE_OK);
117 InitializeSqlite(db);
118 db_.reset(db);
119 }
120
~SqliteEngine()121 SqliteEngine::~SqliteEngine() {
122 // It is important to unregister any functions that have been registered with
123 // the database before destroying it. This is because functions can hold onto
124 // prepared statements, which must be finalized before database destruction.
125 for (auto it = fn_ctx_.GetIterator(); it; ++it) {
126 int ret = sqlite3_create_function_v2(db_.get(), it.key().first.c_str(),
127 it.key().second, SQLITE_UTF8, nullptr,
128 nullptr, nullptr, nullptr, nullptr);
129 if (PERFETTO_UNLIKELY(ret != SQLITE_OK)) {
130 PERFETTO_FATAL("Failed to drop function: '%s'", it.key().first.c_str());
131 }
132 }
133 fn_ctx_.Clear();
134 }
135
PrepareStatement(SqlSource sql)136 SqliteEngine::PreparedStatement SqliteEngine::PrepareStatement(SqlSource sql) {
137 PERFETTO_TP_TRACE(metatrace::Category::QUERY_DETAILED, "QUERY_PREPARE");
138 sqlite3_stmt* raw_stmt = nullptr;
139 int err =
140 sqlite3_prepare_v2(db_.get(), sql.sql().c_str(), -1, &raw_stmt, nullptr);
141 PreparedStatement statement{ScopedStmt(raw_stmt), std::move(sql)};
142 if (err != SQLITE_OK) {
143 const char* errmsg = sqlite3_errmsg(db_.get());
144 std::string frame =
145 statement.sql_source_.AsTracebackForSqliteOffset(GetErrorOffset());
146 base::Status status = base::ErrStatus("%s%s", frame.c_str(), errmsg);
147 status.SetPayload("perfetto.dev/has_traceback", "true");
148
149 statement.status_ = std::move(status);
150 return statement;
151 }
152 if (!raw_stmt) {
153 statement.status_ = base::ErrStatus("No SQL to execute");
154 }
155 return statement;
156 }
157
RegisterFunction(const char * name,int argc,Fn * fn,void * ctx,FnCtxDestructor * destructor,bool deterministic)158 base::Status SqliteEngine::RegisterFunction(const char* name,
159 int argc,
160 Fn* fn,
161 void* ctx,
162 FnCtxDestructor* destructor,
163 bool deterministic) {
164 int flags = SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0);
165 int ret =
166 sqlite3_create_function_v2(db_.get(), name, static_cast<int>(argc), flags,
167 ctx, fn, nullptr, nullptr, destructor);
168 if (ret != SQLITE_OK) {
169 return base::ErrStatus("Unable to register function with name %s", name);
170 }
171 *fn_ctx_.Insert(std::make_pair(name, argc), ctx).first = ctx;
172 return base::OkStatus();
173 }
174
RegisterAggregateFunction(const char * name,int argc,AggregateFnStep * step,AggregateFnFinal * final,void * ctx,FnCtxDestructor * destructor,bool deterministic)175 base::Status SqliteEngine::RegisterAggregateFunction(
176 const char* name,
177 int argc,
178 AggregateFnStep* step,
179 AggregateFnFinal* final,
180 void* ctx,
181 FnCtxDestructor* destructor,
182 bool deterministic) {
183 int flags = SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0);
184 int ret =
185 sqlite3_create_function_v2(db_.get(), name, static_cast<int>(argc), flags,
186 ctx, nullptr, step, final, destructor);
187 if (ret != SQLITE_OK) {
188 return base::ErrStatus("Unable to register function with name %s", name);
189 }
190 return base::OkStatus();
191 }
192
RegisterWindowFunction(const char * name,int argc,WindowFnStep * step,WindowFnInverse * inverse,WindowFnValue * value,WindowFnFinal * final,void * ctx,FnCtxDestructor * destructor,bool deterministic)193 base::Status SqliteEngine::RegisterWindowFunction(const char* name,
194 int argc,
195 WindowFnStep* step,
196 WindowFnInverse* inverse,
197 WindowFnValue* value,
198 WindowFnFinal* final,
199 void* ctx,
200 FnCtxDestructor* destructor,
201 bool deterministic) {
202 int flags = SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0);
203 int ret = sqlite3_create_window_function(
204 db_.get(), name, static_cast<int>(argc), flags, ctx, step, final, value,
205 inverse, destructor);
206 if (ret != SQLITE_OK) {
207 return base::ErrStatus("Unable to register function with name %s", name);
208 }
209 return base::OkStatus();
210 }
211
UnregisterFunction(const char * name,int argc)212 base::Status SqliteEngine::UnregisterFunction(const char* name, int argc) {
213 int ret = sqlite3_create_function_v2(db_.get(), name, static_cast<int>(argc),
214 SQLITE_UTF8, nullptr, nullptr, nullptr,
215 nullptr, nullptr);
216 if (ret != SQLITE_OK) {
217 return base::ErrStatus("Unable to unregister function with name %s", name);
218 }
219 fn_ctx_.Erase({name, argc});
220 return base::OkStatus();
221 }
222
DeclareVirtualTable(const std::string & create_stmt)223 base::Status SqliteEngine::DeclareVirtualTable(const std::string& create_stmt) {
224 int res = sqlite3_declare_vtab(db_.get(), create_stmt.c_str());
225 if (res != SQLITE_OK) {
226 return base::ErrStatus("Declare vtab failed: %s",
227 sqlite3_errmsg(db_.get()));
228 }
229 return base::OkStatus();
230 }
231
GetFunctionContext(const std::string & name,int argc)232 void* SqliteEngine::GetFunctionContext(const std::string& name, int argc) {
233 auto* res = fn_ctx_.Find(std::make_pair(name, argc));
234 return res ? *res : nullptr;
235 }
236
GetErrorOffset() const237 std::optional<uint32_t> SqliteEngine::GetErrorOffset() const {
238 return GetErrorOffsetDb(db_.get());
239 }
240
PreparedStatement(ScopedStmt stmt,SqlSource source)241 SqliteEngine::PreparedStatement::PreparedStatement(ScopedStmt stmt,
242 SqlSource source)
243 : stmt_(std::move(stmt)),
244 expanded_sql_(sqlite3_expanded_sql(stmt_.get())),
245 sql_source_(std::move(source)) {}
246
Step()247 bool SqliteEngine::PreparedStatement::Step() {
248 PERFETTO_TP_TRACE(metatrace::Category::QUERY_DETAILED, "STMT_STEP",
249 [this](metatrace::Record* record) {
250 record->AddArg("Original SQL", original_sql());
251 record->AddArg("Executed SQL", sql());
252 });
253
254 // Now step once into |cur_stmt| so that when we prepare the next statment
255 // we will have executed any dependent bytecode in this one.
256 int err = sqlite3_step(stmt_.get());
257 if (err == SQLITE_ROW) {
258 return true;
259 }
260 if (err == SQLITE_DONE) {
261 return false;
262 }
263 sqlite3* db = sqlite3_db_handle(stmt_.get());
264 std::string frame =
265 sql_source_.AsTracebackForSqliteOffset(GetErrorOffsetDb(db));
266 const char* errmsg = sqlite3_errmsg(db);
267 status_ = base::ErrStatus("%s%s", frame.c_str(), errmsg);
268 return false;
269 }
270
IsDone() const271 bool SqliteEngine::PreparedStatement::IsDone() const {
272 return !sqlite3_stmt_busy(stmt_.get());
273 }
274
original_sql() const275 const char* SqliteEngine::PreparedStatement::original_sql() const {
276 return sql_source_.original_sql().c_str();
277 }
278
sql() const279 const char* SqliteEngine::PreparedStatement::sql() const {
280 return expanded_sql_.get();
281 }
282
283 } // namespace perfetto::trace_processor
284