Implementing uploading of a WebRTC diagnostic log.

components/webrtc_log_uploader/webrtc_log_uploader.cc

+// Copyright (c) 2013 The Chromium Authors. All rights reserved.

[Jói, 2013/05/08 14:14:22]

nit: no (c)

[Henrik Grunell, 2013/05/08 16:05:05]

Removed. Though my conclusion from the bug and the pretty recent thread (started
Feb 20) about this that both are OK (maybe without (c) is preferred). The
presubmit script still allows it, i.e. step 5 in the plan has been skipped.

+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/webrtc_log_uploader/webrtc_log_uploader.h"
+
+#if defined(USE_SYSTEM_LIBBZ2)
+#include <bzlib.h>
+#else
+#include "third_party/bzip2/bzlib.h"
+#endif
+
+#include "base/logging.h"
+#include "base/partial_circular_buffer.h"
+#include "base/shared_memory.h"
+#include "base/stringprintf.h"
+#include "content/public/browser/browser_thread.h"
+#include "net/base/network_delegate.h"
+#include "net/proxy/proxy_config.h"
+#include "net/proxy/proxy_config_service.h"
+#include "net/url_request/url_fetcher.h"
+#include "net/url_request/url_request_context.h"
+#include "net/url_request/url_request_context_builder.h"
+#include "net/url_request/url_request_context_getter.h"
+
+namespace components {
+
+namespace {
+
+const uint32 kIntermediateCompressionBufferSize = 256 * 1024;  // 256 KB
+
+const char kTempUploadFile[] = "/tmp/chromium-webrtc-log-upload";

[Jói, 2013/05/08 14:14:22]

This isn't going to work on all platforms.  I think you want
file_util::CreateTemporaryFile.

[Henrik Grunell, 2013/05/08 16:05:05]

Done.

+const char kUploadURL[] = "https://clients2.google.com/cr/report";

[Jói, 2013/05/08 14:14:22]

Is this a new API? It should probably require an API key to be included with the
request, to help control abuse. I'll send you details on that.

[Henrik Grunell, 2013/05/08 16:05:05]

This is the ordinary crash server. See chrome/app/breakpad_linux.cc and
chrome/tools/crash_service/crash_service.cc

+const char kUploadContentType[] = "multipart/form-data";
+const char kMultipartBoundary[] =
+    "----**--yradnuoBgoLtrapitluMklaTelgooG--**----";
+
+// Config getter that always returns direct settings.
+class ProxyConfigServiceDirect : public net::ProxyConfigService {
+ public:
+  // ProxyConfigService implementation.
+  virtual void AddObserver(Observer* observer) OVERRIDE {}
+  virtual void RemoveObserver(Observer* observer) OVERRIDE {}
+  virtual ConfigAvailability GetLatestProxyConfig(
+      net::ProxyConfig* config) OVERRIDE {
+    *config = net::ProxyConfig::CreateDirect();
+    return CONFIG_VALID;
+  }
+};
+
+}  // namespace
+
+class WebRtcLogURLRequestContextGetter : public net::URLRequestContextGetter {
+ public:
+  WebRtcLogURLRequestContextGetter() {}
+
+  // net::URLRequestContextGetter implementation.
+  virtual net::URLRequestContext* GetURLRequestContext() OVERRIDE {
+    DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::IO));
+
+    if (!url_request_context_) {
+      net::URLRequestContextBuilder builder;
+#if defined(OS_LINUX) || defined(OS_ANDROID)
+      builder.set_proxy_config_service(new ProxyConfigServiceDirect());
+#endif
+      url_request_context_.reset(builder.Build());
+    }
+    CHECK(url_request_context_.get());
+
+    return url_request_context_.get();
+  }
+
+  virtual scoped_refptr<base::SingleThreadTaskRunner>
+      GetNetworkTaskRunner() const OVERRIDE {
+    return content::BrowserThread::GetMessageLoopProxyForThread(
+        content::BrowserThread::IO);
+  }
+
+ private:
+  virtual ~WebRtcLogURLRequestContextGetter() {}
+
+  // NULL if not yet initialized. Otherwise, it is the URLRequestContext
+  // instance that was lazily created by GetURLRequestContext().
+  // Access only from the IO thread.
+  scoped_ptr<net::URLRequestContext> url_request_context_;
+
+  DISALLOW_COPY_AND_ASSIGN(WebRtcLogURLRequestContextGetter);
+};
+
+
+WebRtcLogUploader::WebRtcLogUploader()
+    : multipart_file_(base::kInvalidPlatformFileValue) {
+}
+
+WebRtcLogUploader::~WebRtcLogUploader() {
+}
+
+void WebRtcLogUploader::OnURLFetchComplete(
+    const net::URLFetcher* source) {
+  std::vector<net::URLFetcher*>::iterator it;
+  for (it = url_fetchers_.begin(); it != url_fetchers_.end(); ++it) {
+    if (*it == source) {
+      url_fetchers_.erase(it);
+      break;
+    }
+  }
+  delete source;
+}
+
+void WebRtcLogUploader::OnURLFetchUploadProgress(
+    const net::URLFetcher* source, int64 current, int64 total) {
+}
+
+void WebRtcLogUploader::UploadLog(base::SharedMemory* shared_memory,
+                                  uint32 length) {
+  DCHECK(content::BrowserThread::CurrentlyOn(content::BrowserThread::FILE));
+  DCHECK(shared_memory);
+  DCHECK(shared_memory->memory());
+
+  // We have taken ownership of |shared_memory|, make sure it's deleted when
+  // we're done.
+  // TODO(grunell): Remove when passing a scoped_ptr instead.
+  scoped_ptr<base::SharedMemory> scoped_shared_memory(shared_memory);
+
+  SetupMultipartFile(reinterpret_cast<uint8*>(scoped_shared_memory->memory()),
+                     length);
+
+  if (!request_context_getter_)
+    request_context_getter_ = new WebRtcLogURLRequestContextGetter();
+
+  std::string content_type = kUploadContentType;
+  content_type.append("; boundary=");
+  content_type.append(kMultipartBoundary);
+
+  net::URLFetcher* url_fetcher =
+      net::URLFetcher::Create(GURL(kUploadURL), net::URLFetcher::POST, this);
+  url_fetcher->SetRequestContext(request_context_getter_);
+  url_fetcher->SetUploadFilePath(
+      content_type, base::FilePath(kTempUploadFile), 0, kuint64max,
+      content::BrowserThread::GetMessageLoopProxyForThread(
+          content::BrowserThread::FILE));
+  url_fetcher->Start();
+  url_fetchers_.push_back(url_fetcher);
+}
+
+void WebRtcLogUploader::SetupMultipartFile(uint8* log_buffer,

[Jói, 2013/05/08 14:14:22]

Is it possible to reuse existing code to create a multipart file?  A quick grep
of the //net code for "multipart" seems to indicate we might only have multipart
support there for downloads, but I think it would be worth checking with e.g.
wtc@ to see if there is opportunity for reuse.

If reuse is not possible, then someone from the net team familiar with the the
HTTP multipart format should review this implementation; again wtc@ could
probably help find an appropriate reviewer.

This code should have unit tests, but wait to add them until you've resolved
whether you can reuse code or not.

[Henrik Grunell, 2013/05/08 16:05:05]

Yeah, couldn't really find anything reusable when I checked some time ago. I'll
check with wtc@.

+                                           uint32 log_buffer_length) {
+  int flags = base::PLATFORM_FILE_CREATE_ALWAYS |
+              base::PLATFORM_FILE_WRITE;
+  bool created = false;
+  base::PlatformFileError error = base::PLATFORM_FILE_OK;
+  // TODO(grunell): Handle several logs. (Different random file names etc.)
+  multipart_file_ =
+      CreatePlatformFile(base::FilePath(kTempUploadFile), flags,
+                         &created, &error);
+  DCHECK(created);
+  DCHECK(error == base::PLATFORM_FILE_OK);
+
+  // TODO(grunell): Correct product name.
+  AddPairString("prod", "Chrome");
+  // TODO(grunell): Correct version.
+  AddPairString("ver", "0.0.1.1-dev-test");
+  AddPairString("guid", "0");
+  AddPairString("type", "log");
+
+  AddUrlChunks();
+
+  AddLogData(log_buffer, log_buffer_length);
+
+  std::stringstream ss;
+  ss << "--" << kMultipartBoundary << "--" << "\r\n";
+  base::WritePlatformFileAtCurrentPos(
+      multipart_file_, ss.str().data(), ss.str().size());
+
+  DCHECK(base::ClosePlatformFile(multipart_file_));
+}
+
+void WebRtcLogUploader::AddPairString(const std::string& key,
+                                             const std::string& value) {
+  std::string str;
+  AddMultipartValueForUpload(key, value, kMultipartBoundary, "", &str);
+  base::WritePlatformFileAtCurrentPos(multipart_file_, str.c_str(), str.size());
+}
+
+void WebRtcLogUploader::AddUrlChunks() {
+  // TODO(grunell): Implement.
+}
+
+void WebRtcLogUploader::AddLogData(uint8* log_buffer,
+                                   uint32 log_buffer_length) {
+  std::stringstream ss;
+  ss << "--" << kMultipartBoundary << "\r\n";
+  ss << "Content-Disposition: form-data; name=\"log\"";
+  ss << "; filename=\"log.bz2\"" << "\r\n";
+  ss << "Content-Type: application/x-bzip" << "\r\n";
+  ss << "\r\n";
+  base::WritePlatformFileAtCurrentPos(
+      multipart_file_, ss.str().data(), ss.str().size());
+
+  CompressLog(log_buffer, log_buffer_length, multipart_file_);
+
+  std::string end_str = "\r\n";
+  base::WritePlatformFileAtCurrentPos(
+      multipart_file_, end_str.c_str(), end_str.size());
+}
+
+// TODO(grunell): This is copied from cloud_print_helpers.cc. Break out to its
+// own file.
+void WebRtcLogUploader::AddMultipartValueForUpload(
+    const std::string& value_name, const std::string& value,
+    const std::string& mime_boundary, const std::string& content_type,
+    std::string* post_data) {
+  DCHECK(post_data);
+  // First line is the boundary
+  post_data->append("--" + mime_boundary + "\r\n");
+  // Next line is the Content-disposition
+  post_data->append(base::StringPrintf("Content-Disposition: form-data; "
+                   "name=\"%s\"\r\n", value_name.c_str()));
+  if (!content_type.empty()) {
+    // If Content-type is specified, the next line is that
+    post_data->append(base::StringPrintf("Content-Type: %s\r\n",
+                      content_type.c_str()));
+  }
+  // Leave an empty line and append the value.
+  post_data->append(base::StringPrintf("\r\n%s\r\n", value.c_str()));
+}
+
+void WebRtcLogUploader::CompressLog(uint8* input,
+                                    uint32 input_size,
+                                    base::PlatformFile output_file) {
+  // TODO(grunell): Remove debugging code.
+  const bool debug_print = false;
+
+  if (debug_print) LOG(ERROR) << "*** DUMPING LOG BUFFER BEGIN ***";
+
+  base::PartialCircularBuffer read_pcb(input, input_size);
+
+  bz_stream stream = {0};
+  int result = BZ2_bzCompressInit(&stream, 3, 0, 0);
+  DCHECK(result == BZ_OK);
+
+  scoped_ptr<uint8[]> intermediate_buffer(
+      new uint8[kIntermediateCompressionBufferSize]);
+  scoped_ptr<uint8[]> compressed(
+      new uint8[kIntermediateCompressionBufferSize]);
+  memset(intermediate_buffer.get(), 0, kIntermediateCompressionBufferSize);
+  memset(compressed.get(), 0, kIntermediateCompressionBufferSize);
+  uint32 read = read_pcb.Read(intermediate_buffer.get(),

[Jói, 2013/05/08 14:14:22]

This and the stream.next_in, stream.avail_in setup below seems to be duplicated
in the while loop; can you avoid duplication if you make it a do/while loop?

[Henrik Grunell, 2013/05/08 16:05:05]

In case the intermediate buffer isn't filled, that is we won't have any more
data that this to compress more than this, my understanding is that we should go
directly to the BZ_FINISH action. The documentation isn't entirely clear if one
BZ_RUN action is OK, but that's how I interpret it.

Do you prefer a do-while loop with a break in it?

[Jói, 2013/05/08 17:00:31]

I see.  I think a do-while loop with a break will avoid the code duplication,
which makes it easier to read and maintain.  You might want to document that
you're intentionally skipping the BZ_RUN if BZ_FINISH would be the next action,
and rewriting the loop this way will give you a natural way to document that (at
the break).

[Henrik Grunell, 2013/05/15 20:17:53]

Done.

+                              kIntermediateCompressionBufferSize);
+
+  if (debug_print) {
+    std::string tmp(
+        reinterpret_cast<char*>(&intermediate_buffer.get()[0]), read);
+    LOG(ERROR) << "Data (" << read << "):" << '\n' << tmp;
+  }
+
+  stream.next_in = reinterpret_cast<char*>(&intermediate_buffer.get()[0]);
+  stream.avail_in = read;
+  char* compressed_char_ptr = reinterpret_cast<char*>(&compressed.get()[0]);
+
+  // Keeps track of written output.
+  unsigned int last_total_out = stream.total_out_lo32;
+
+  while (read == kIntermediateCompressionBufferSize) {
+    stream.next_out = compressed_char_ptr;
+    stream.avail_out = kIntermediateCompressionBufferSize;
+    result = BZ2_bzCompress(&stream, BZ_RUN);
+    if (debug_print) {
+      LOG(ERROR) << "RESULT = " << result << ", read = " << read;
+      LOG(ERROR) << "avail_in = " << stream.avail_in
+                 << ", total_in_lo32 = " << stream.total_in_lo32;
+      LOG(ERROR) << "avail_out = " << stream.avail_out
+                 << ", total_out_lo32 = " << stream.total_out_lo32;
+    }
+    DCHECK(result == BZ_RUN_OK);
+    base::WritePlatformFileAtCurrentPos(output_file, compressed_char_ptr,
+                                        stream.total_out_lo32 - last_total_out);
+    last_total_out = stream.total_out_lo32;
+    read = read_pcb.Read(intermediate_buffer.get(),
+                         kIntermediateCompressionBufferSize);
+    stream.next_in = reinterpret_cast<char*>(&intermediate_buffer.get()[0]);
+    stream.avail_in = read;
+
+    if (debug_print) {
+      std::string tmp(
+          reinterpret_cast<char*>(&intermediate_buffer.get()[0]), read);
+      LOG(ERROR) << "Data (" << read << "):" << '\n' << tmp;
+    }
+  }
+
+  do {
+    stream.next_out = compressed_char_ptr;
+    stream.avail_out = kIntermediateCompressionBufferSize;
+    result = BZ2_bzCompress(&stream, BZ_FINISH);
+    if (debug_print) {
+      LOG(ERROR) << "RESULT = " << result << ", read = " << read;
+      LOG(ERROR) << "avail_in = " << stream.avail_in
+                 << ", total_in_lo32 = " << stream.total_in_lo32;
+      LOG(ERROR) << "avail_out = " << stream.avail_out
+                 << ", total_out_lo32 = " << stream.total_out_lo32;
+    }
+    base::WritePlatformFileAtCurrentPos(output_file, compressed_char_ptr,
+                                        stream.total_out_lo32 - last_total_out);
+    last_total_out = stream.total_out_lo32;
+  } while (result == BZ_FINISH_OK);
+  DCHECK(result == BZ_STREAM_END);
+
+  result = BZ2_bzCompressEnd(&stream);
+  DCHECK(result == BZ_OK);
+
+  if (debug_print) LOG(ERROR) << "*** DUMPING LOG BUFFER END ***";
+}
+
+}  // namespace components