1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc. All rights reserved.
3 // https://developers.google.com/protocol-buffers/
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 // * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 // * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 // Author: [email protected] (Brian Olson)
32 //
33 // This file contains the implementation of classes GzipInputStream and
34 // GzipOutputStream.
35
36
37 #if HAVE_ZLIB
38 #include <google/protobuf/io/gzip_stream.h>
39 #include <google/protobuf/port.h>
40
41 #include <google/protobuf/stubs/common.h>
42 #include <google/protobuf/stubs/logging.h>
43
44 namespace google {
45 namespace protobuf {
46 namespace io {
47
48 static const int kDefaultBufferSize = 65536;
49
GzipInputStream(ZeroCopyInputStream * sub_stream,Format format,int buffer_size)50 GzipInputStream::GzipInputStream(ZeroCopyInputStream* sub_stream, Format format,
51 int buffer_size)
52 : format_(format), sub_stream_(sub_stream), zerror_(Z_OK), byte_count_(0) {
53 zcontext_.state = Z_NULL;
54 zcontext_.zalloc = Z_NULL;
55 zcontext_.zfree = Z_NULL;
56 zcontext_.opaque = Z_NULL;
57 zcontext_.total_out = 0;
58 zcontext_.next_in = NULL;
59 zcontext_.avail_in = 0;
60 zcontext_.total_in = 0;
61 zcontext_.msg = NULL;
62 if (buffer_size == -1) {
63 output_buffer_length_ = kDefaultBufferSize;
64 } else {
65 output_buffer_length_ = buffer_size;
66 }
67 output_buffer_ = operator new(output_buffer_length_);
68 GOOGLE_CHECK(output_buffer_ != NULL);
69 zcontext_.next_out = static_cast<Bytef*>(output_buffer_);
70 zcontext_.avail_out = output_buffer_length_;
71 output_position_ = output_buffer_;
72 }
~GzipInputStream()73 GzipInputStream::~GzipInputStream() {
74 internal::SizedDelete(output_buffer_, output_buffer_length_);
75 zerror_ = inflateEnd(&zcontext_);
76 }
77
internalInflateInit2(z_stream * zcontext,GzipInputStream::Format format)78 static inline int internalInflateInit2(z_stream* zcontext,
79 GzipInputStream::Format format) {
80 int windowBitsFormat = 0;
81 switch (format) {
82 case GzipInputStream::GZIP:
83 windowBitsFormat = 16;
84 break;
85 case GzipInputStream::AUTO:
86 windowBitsFormat = 32;
87 break;
88 case GzipInputStream::ZLIB:
89 windowBitsFormat = 0;
90 break;
91 }
92 return inflateInit2(zcontext, /* windowBits */ 15 | windowBitsFormat);
93 }
94
Inflate(int flush)95 int GzipInputStream::Inflate(int flush) {
96 if ((zerror_ == Z_OK) && (zcontext_.avail_out == 0)) {
97 // previous inflate filled output buffer. don't change input params yet.
98 } else if (zcontext_.avail_in == 0) {
99 const void* in;
100 int in_size;
101 bool first = zcontext_.next_in == NULL;
102 bool ok = sub_stream_->Next(&in, &in_size);
103 if (!ok) {
104 zcontext_.next_out = NULL;
105 zcontext_.avail_out = 0;
106 return Z_STREAM_END;
107 }
108 zcontext_.next_in = static_cast<Bytef*>(const_cast<void*>(in));
109 zcontext_.avail_in = in_size;
110 if (first) {
111 int error = internalInflateInit2(&zcontext_, format_);
112 if (error != Z_OK) {
113 return error;
114 }
115 }
116 }
117 zcontext_.next_out = static_cast<Bytef*>(output_buffer_);
118 zcontext_.avail_out = output_buffer_length_;
119 output_position_ = output_buffer_;
120 int error = inflate(&zcontext_, flush);
121 return error;
122 }
123
DoNextOutput(const void ** data,int * size)124 void GzipInputStream::DoNextOutput(const void** data, int* size) {
125 *data = output_position_;
126 *size = ((uintptr_t)zcontext_.next_out) - ((uintptr_t)output_position_);
127 output_position_ = zcontext_.next_out;
128 }
129
130 // implements ZeroCopyInputStream ----------------------------------
Next(const void ** data,int * size)131 bool GzipInputStream::Next(const void** data, int* size) {
132 bool ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) ||
133 (zerror_ == Z_BUF_ERROR);
134 if ((!ok) || (zcontext_.next_out == NULL)) {
135 return false;
136 }
137 if (zcontext_.next_out != output_position_) {
138 DoNextOutput(data, size);
139 return true;
140 }
141 if (zerror_ == Z_STREAM_END) {
142 if (zcontext_.next_out != NULL) {
143 // sub_stream_ may have concatenated streams to follow
144 zerror_ = inflateEnd(&zcontext_);
145 byte_count_ += zcontext_.total_out;
146 if (zerror_ != Z_OK) {
147 return false;
148 }
149 zerror_ = internalInflateInit2(&zcontext_, format_);
150 if (zerror_ != Z_OK) {
151 return false;
152 }
153 } else {
154 *data = NULL;
155 *size = 0;
156 return false;
157 }
158 }
159 zerror_ = Inflate(Z_NO_FLUSH);
160 if ((zerror_ == Z_STREAM_END) && (zcontext_.next_out == NULL)) {
161 // The underlying stream's Next returned false inside Inflate.
162 return false;
163 }
164 ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) ||
165 (zerror_ == Z_BUF_ERROR);
166 if (!ok) {
167 return false;
168 }
169 DoNextOutput(data, size);
170 return true;
171 }
BackUp(int count)172 void GzipInputStream::BackUp(int count) {
173 output_position_ = reinterpret_cast<void*>(
174 reinterpret_cast<uintptr_t>(output_position_) - count);
175 }
Skip(int count)176 bool GzipInputStream::Skip(int count) {
177 const void* data;
178 int size = 0;
179 bool ok = Next(&data, &size);
180 while (ok && (size < count)) {
181 count -= size;
182 ok = Next(&data, &size);
183 }
184 if (size > count) {
185 BackUp(size - count);
186 }
187 return ok;
188 }
ByteCount() const189 int64_t GzipInputStream::ByteCount() const {
190 int64_t ret = byte_count_ + zcontext_.total_out;
191 if (zcontext_.next_out != NULL && output_position_ != NULL) {
192 ret += reinterpret_cast<uintptr_t>(zcontext_.next_out) -
193 reinterpret_cast<uintptr_t>(output_position_);
194 }
195 return ret;
196 }
197
198 // =========================================================================
199
Options()200 GzipOutputStream::Options::Options()
201 : format(GZIP),
202 buffer_size(kDefaultBufferSize),
203 compression_level(Z_DEFAULT_COMPRESSION),
204 compression_strategy(Z_DEFAULT_STRATEGY) {}
205
GzipOutputStream(ZeroCopyOutputStream * sub_stream)206 GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream) {
207 Init(sub_stream, Options());
208 }
209
GzipOutputStream(ZeroCopyOutputStream * sub_stream,const Options & options)210 GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream,
211 const Options& options) {
212 Init(sub_stream, options);
213 }
214
Init(ZeroCopyOutputStream * sub_stream,const Options & options)215 void GzipOutputStream::Init(ZeroCopyOutputStream* sub_stream,
216 const Options& options) {
217 sub_stream_ = sub_stream;
218 sub_data_ = NULL;
219 sub_data_size_ = 0;
220
221 input_buffer_length_ = options.buffer_size;
222 input_buffer_ = operator new(input_buffer_length_);
223 GOOGLE_CHECK(input_buffer_ != NULL);
224
225 zcontext_.zalloc = Z_NULL;
226 zcontext_.zfree = Z_NULL;
227 zcontext_.opaque = Z_NULL;
228 zcontext_.next_out = NULL;
229 zcontext_.avail_out = 0;
230 zcontext_.total_out = 0;
231 zcontext_.next_in = NULL;
232 zcontext_.avail_in = 0;
233 zcontext_.total_in = 0;
234 zcontext_.msg = NULL;
235 // default to GZIP format
236 int windowBitsFormat = 16;
237 if (options.format == ZLIB) {
238 windowBitsFormat = 0;
239 }
240 zerror_ =
241 deflateInit2(&zcontext_, options.compression_level, Z_DEFLATED,
242 /* windowBits */ 15 | windowBitsFormat,
243 /* memLevel (default) */ 8, options.compression_strategy);
244 }
245
~GzipOutputStream()246 GzipOutputStream::~GzipOutputStream() {
247 Close();
248 internal::SizedDelete(input_buffer_, input_buffer_length_);
249 }
250
251 // private
Deflate(int flush)252 int GzipOutputStream::Deflate(int flush) {
253 int error = Z_OK;
254 do {
255 if ((sub_data_ == NULL) || (zcontext_.avail_out == 0)) {
256 bool ok = sub_stream_->Next(&sub_data_, &sub_data_size_);
257 if (!ok) {
258 sub_data_ = NULL;
259 sub_data_size_ = 0;
260 return Z_BUF_ERROR;
261 }
262 GOOGLE_CHECK_GT(sub_data_size_, 0);
263 zcontext_.next_out = static_cast<Bytef*>(sub_data_);
264 zcontext_.avail_out = sub_data_size_;
265 }
266 error = deflate(&zcontext_, flush);
267 } while (error == Z_OK && zcontext_.avail_out == 0);
268 if ((flush == Z_FULL_FLUSH) || (flush == Z_FINISH)) {
269 // Notify lower layer of data.
270 sub_stream_->BackUp(zcontext_.avail_out);
271 // We don't own the buffer anymore.
272 sub_data_ = NULL;
273 sub_data_size_ = 0;
274 }
275 return error;
276 }
277
278 // implements ZeroCopyOutputStream ---------------------------------
Next(void ** data,int * size)279 bool GzipOutputStream::Next(void** data, int* size) {
280 if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) {
281 return false;
282 }
283 if (zcontext_.avail_in != 0) {
284 zerror_ = Deflate(Z_NO_FLUSH);
285 if (zerror_ != Z_OK) {
286 return false;
287 }
288 }
289 if (zcontext_.avail_in == 0) {
290 // all input was consumed. reset the buffer.
291 zcontext_.next_in = static_cast<Bytef*>(input_buffer_);
292 zcontext_.avail_in = input_buffer_length_;
293 *data = input_buffer_;
294 *size = input_buffer_length_;
295 } else {
296 // The loop in Deflate should consume all avail_in
297 GOOGLE_LOG(DFATAL) << "Deflate left bytes unconsumed";
298 }
299 return true;
300 }
BackUp(int count)301 void GzipOutputStream::BackUp(int count) {
302 GOOGLE_CHECK_GE(zcontext_.avail_in, static_cast<uInt>(count));
303 zcontext_.avail_in -= count;
304 }
ByteCount() const305 int64_t GzipOutputStream::ByteCount() const {
306 return zcontext_.total_in + zcontext_.avail_in;
307 }
308
Flush()309 bool GzipOutputStream::Flush() {
310 zerror_ = Deflate(Z_FULL_FLUSH);
311 // Return true if the flush succeeded or if it was a no-op.
312 return (zerror_ == Z_OK) ||
313 (zerror_ == Z_BUF_ERROR && zcontext_.avail_in == 0 &&
314 zcontext_.avail_out != 0);
315 }
316
Close()317 bool GzipOutputStream::Close() {
318 if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) {
319 return false;
320 }
321 do {
322 zerror_ = Deflate(Z_FINISH);
323 } while (zerror_ == Z_OK);
324 zerror_ = deflateEnd(&zcontext_);
325 bool ok = zerror_ == Z_OK;
326 zerror_ = Z_STREAM_END;
327 return ok;
328 }
329
330 } // namespace io
331 } // namespace protobuf
332 } // namespace google
333
334 #endif // HAVE_ZLIB
335