1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
|
# name: test/sql/logging/logging_csv.test
# description: Test csv formatted log storages (stdout & file)
# group: [logging]
require noforcestorage
# Due to different file locking behaviour, this currently fails on windows
require notwindows
# Enable FileSystem logging to single csv file
statement ok
CALL enable_logging(['FileSystem'], storage='file', storage_config={'path': '{TEMP_DIR}/logging_csv_log.csv', 'normalize': false});
# Read some data to trigger FileSystem log
statement ok
FROM "{DATA_DIR}/csv/big_number.csv"
query IIIIII
DESCRIBE FROM '{TEMP_DIR}/logging_csv_log.csv';
----
context_id BIGINT YES NULL NULL NULL
scope VARCHAR YES NULL NULL NULL
connection_id BIGINT YES NULL NULL NULL
transaction_id BIGINT YES NULL NULL NULL
query_id BIGINT YES NULL NULL NULL
thread_id VARCHAR YES NULL NULL NULL
timestamp TIMESTAMP WITH TIME ZONE YES NULL NULL NULL
type VARCHAR YES NULL NULL NULL
log_level VARCHAR YES NULL NULL NULL
message VARCHAR YES NULL NULL NULL
# Ensure we can reparse the structured log message from the csv
query III
SELECT
scope,
path: parse_duckdb_log_message('FileSystem', message)['path'],
op: parse_duckdb_log_message('FileSystem', message)['op'],
FROM "{TEMP_DIR}/logging_csv_log.csv"
WHERE path = '{DATA_DIR}/csv/big_number.csv';
----
CONNECTION {DATA_DIR}/csv/big_number.csv OPEN
CONNECTION {DATA_DIR}/csv/big_number.csv READ
CONNECTION {DATA_DIR}/csv/big_number.csv READ
CONNECTION {DATA_DIR}/csv/big_number.csv CLOSE
statement ok
CALL disable_logging()
# Truncating the logs will clear the csv log file
statement ok
CALL truncate_duckdb_logs();
query I
select count(*) FROM "{TEMP_DIR}/logging_csv_log.csv";
----
0
# Enable FileSystem logging to normalized files
statement ok
CALL enable_logging(['FileSystem'], storage='file', storage_config={'path': '{TEMP_DIR}/logging_csv_logs_normalized', 'normalize': true});
# Read some data to trigger FileSystem log
statement ok
FROM "{DATA_DIR}/csv/big_number.csv"
# Ensure we can reparse the structured log message from the csv
query III
SELECT
context_id is not null,
path: parse_duckdb_log_message('FileSystem', message)['path'],
op: parse_duckdb_log_message('FileSystem', message)['op'],
FROM "{TEMP_DIR}/logging_csv_logs_normalized/duckdb_log_entries.csv"
WHERE path = '{DATA_DIR}/csv/big_number.csv';
----
1 {DATA_DIR}/csv/big_number.csv OPEN
1 {DATA_DIR}/csv/big_number.csv READ
1 {DATA_DIR}/csv/big_number.csv READ
1 {DATA_DIR}/csv/big_number.csv CLOSE
# Contexts are now in a separate csv file
# TODO: is this correct?
query I
SELECT scope
FROM "{TEMP_DIR}/logging_csv_logs_normalized/duckdb_log_contexts.csv";
----
CONNECTION
CONNECTION
# Check schema
query IIIIII
DESCRIBE FROM '{TEMP_DIR}/logging_csv_logs_normalized/duckdb_log_entries.csv';
----
context_id BIGINT YES NULL NULL NULL
timestamp TIMESTAMP WITH TIME ZONE YES NULL NULL NULL
type VARCHAR YES NULL NULL NULL
log_level VARCHAR YES NULL NULL NULL
message VARCHAR YES NULL NULL NULL
# Check schema
query IIIIII
DESCRIBE FROM '{TEMP_DIR}/logging_csv_logs_normalized/duckdb_log_contexts.csv';
----
context_id BIGINT YES NULL NULL NULL
scope VARCHAR YES NULL NULL NULL
connection_id BIGINT YES NULL NULL NULL
transaction_id BIGINT YES NULL NULL NULL
query_id BIGINT YES NULL NULL NULL
thread_id VARCHAR YES NULL NULL NULL
statement ok
CALL disable_logging();
# Truncating the logs will clear both csv files
statement ok
CALL truncate_duckdb_logs();
query I
select count(*) FROM "{TEMP_DIR}/logging_csv_logs_normalized/duckdb_log_contexts.csv";
----
0
query I
select count(*) FROM "{TEMP_DIR}/logging_csv_logs_normalized/duckdb_log_entries.csv";
----
0
statement ok
CALL enable_logging(['FileSystem'], storage='stdout');
# TODO: we can't nicely test logging to stdout without polluting test runner output
statement ok
CALL truncate_duckdb_logs();
statement ok
CALL disable_logging();
# Both stdout and file logging have a buffer size param which controls when the buffered csv data is written out. This is currently hard to test though since
# we flush these buffers after every query anyways
statement ok
CALL enable_logging(['FileSystem'], storage='stdout', storage_config={'buffer_size': 1000000});
statement ok
CALL disable_logging();
# Try some invalid configs ensuring they throw nice errors
statement error
CALL enable_logging(['FileSystem'], storage='stdout', storage_config={'bla': 'bla'});
----
Invalid Input Error: Unrecognized log storage config option for storage: 'StdOutLogStorage': 'bla'
statement error
CALL enable_logging(['FileSystem'], storage='stdout', storage_config={'path': './file.csv'});
----
Invalid Input Error: Unrecognized log storage config option for storage: 'StdOutLogStorage': 'path'
# Switching between normalized and denormalized logs will throw if the log is non-empty
statement ok
CALL truncate_duckdb_logs()
statement ok
CALL enable_logging(['QueryLog'], storage='file', storage_config={'path': '{TEMP_DIR}/logging_csv_log.csv'});
statement ok
SELECT 1;
statement ok
CALL truncate_duckdb_logs()
# statement error
# CALL enable_logging(['QueryLog'], storage='file', storage_config={'entries_path': '{TEMP_DIR}/logging_csv_log_entries.csv', 'contexts_path': '{TEMP_DIR}/logging_csv_log_contexts.csv'});
# ----
# Invalid Configuration Error: Cannot change between normalized and denormalized with a non-empty log. Please truncate the log first
statement ok
explain FROM duckdb_logs
# This is not allowed
statement error
CALL enable_logging(['QueryLog'], storage='file', storage_config={'path': '{TEMP_DIR}/logging_csv_log.csv', 'normalize': true});
----
Invalid Configuration Error: Can not set path to '
statement error
CALL enable_logging(['QueryLog'], storage='file', storage_config={'path': '{TEMP_DIR}/logging_csv_log.csv', 'normalize': true});
----
' while normalize is true. Normalize will make DuckDB write multiple log files to more efficiently store log entries. Please specify a directory path instead of a csv file path, or set normalize to false.
statement ok
CALL truncate_duckdb_logs();
statement ok
CALL disable_logging();
# Test switching CSV delimiters
statement ok
CALL enable_logging(['QueryLog'], storage='file', storage_config={'path': '{TEMP_DIR}/logging_csv_log_delim.csv', 'delim': ';'});
statement ok
SELECT 1;
query I
SELECT message FROM read_csv('{TEMP_DIR}/logging_csv_log_delim.csv', delim=';');
----
SELECT 1
|
