protobuf如何还原proto源文件及描述字符串中左括弧的意义

语法分析

和通常的有语法的结构一样，proto的编译也经过词法(tokenize)和语法(parse)两个阶段，相关代码分别在tokenizer.cc和parser.cc两个文件中。

/// @file: protobuf\src\google\protobuf\compiler\parser.cc
bool Parser::ParseTopLevelStatement(FileDescriptorProto* file,
                                    const LocationRecorder& root_location) {
  if (TryConsumeEndOfDeclaration(";", NULL)) {
    // empty statement; ignore
    return true;
  } else if (LookingAt("message")) {
    LocationRecorder location(root_location,
                              FileDescriptorProto::kMessageTypeFieldNumber,
                              file->message_type_size());
    return ParseMessageDefinition(file->add_message_type(), location, file);
  } else if (LookingAt("enum")) {
    LocationRecorder location(root_location,
                              FileDescriptorProto::kEnumTypeFieldNumber,
                              file->enum_type_size());
    return ParseEnumDefinition(file->add_enum_type(), location, file);
  } else if (LookingAt("service")) {
    LocationRecorder location(root_location,
                              FileDescriptorProto::kServiceFieldNumber,
                              file->service_size());
    return ParseServiceDefinition(file->add_service(), location, file);
  } else if (LookingAt("extend")) {
    LocationRecorder location(root_location,
                              FileDescriptorProto::kExtensionFieldNumber);
    return ParseExtend(
        file->mutable_extension(), file->mutable_message_type(), root_location,
        FileDescriptorProto::kMessageTypeFieldNumber, location, file);
  } else if (LookingAt("import")) {
    return ParseImport(file->mutable_dependency(),
                       file->mutable_public_dependency(),
                       file->mutable_weak_dependency(), root_location, file);
  } else if (LookingAt("package")) {
    return ParsePackage(file, root_location, file);
  } else if (LookingAt("option")) {
    LocationRecorder location(root_location,
                              FileDescriptorProto::kOptionsFieldNumber);
    return ParseOption(file->mutable_options(), location, file,
                       OPTION_STATEMENT);
  } else {
    AddError("Expected top-level statement (e.g. \"message\").");
    return false;
  }
}

DescriptorTable

每个proto文件生成的cpp文件中都有一串特别醒目的字符串，该字符串位于一个DescriptorTable结构的const char* descriptor字段，它也是整个proto文件生成FileDescriptorProto结构序列化之后的内容。

/// @file: protobuf\src\google\protobuf\generated_message_reflection.h

// This struct tries to reduce unnecessary padding.
// The num_xxx might not be close to their respective pointer, but this saves
// padding.
struct PROTOBUF_EXPORT DescriptorTable {
  mutable bool is_initialized;
  bool is_eager;
  int size;  // of serialized descriptor
  const char* descriptor;
  const char* filename;
  once_flag* once;
  const DescriptorTable* const* deps;
  int num_deps;
  int num_messages;
  const MigrationSchema* schemas;
  const Message* const* default_instances;
  const uint32* offsets;
  // update the following descriptor arrays.
  Metadata* file_level_metadata;
  const EnumDescriptor** file_level_enum_descriptors;
  const ServiceDescriptor** file_level_service_descriptors;
};

生成的cpp代码

在生成的cpp文件中类似于2e这种莫名其妙的字符串是从哪里来的呢？例如这个变量名assign_descriptors_table_tsecer_2eproto中的2e，从代码实现看也很简单，就是把不属于数字字母(ascii_isalnum)的字符转换为""+16进制字符串。所以assign_descriptors_table_tsecer_2eproto中的"2e"就是由于"tsecer.proto"中的"."字符内码"2e"加上""组成；同样，如果是文件名中有""，那么会转换为"_5f"。

/// @file: protobuf\src\google\protobuf\compiler\cpp\cpp_helpers.cc
// Convert a file name into a valid identifier.
std::string FilenameIdentifier(const std::string& filename) {
  std::string result;
  for (int i = 0; i < filename.size(); i++) {
    if (ascii_isalnum(filename[i])) {
      result.push_back(filename[i]);
    } else {
      // Not alphanumeric.  To avoid any possibility of name conflicts we
      // use the hex code for the character.
      StrAppend(&result, "_",
                      strings::Hex(static_cast<uint8_t>(filename[i])));
    }
  }
  return result;
}

std::string UniqueName(const std::string& name, const std::string& filename,
                       const Options& options) {
  return name + "_" + FilenameIdentifier(filename);
}
///@file: protobuf\src\google\protobuf\compiler\cpp\cpp_file.cc
void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) {
  Formatter format(printer, variables_);
///....
  // Embed the descriptor.  We simply serialize the entire
  // FileDescriptorProto/ and embed it as a string literal, which is parsed and
  // built into real descriptors at initialization time.
  const std::string protodef_name =
      UniqueName("descriptor_table_protodef", file_, options_);
///...
  // The DescriptorTable itself.
  // Should be "bool eager = NeedsEagerDescriptorAssignment(file_, options_);"
  // however this might cause a tsan failure in superroot b/148382879,
  // so disable for now.
  bool eager = false;
  format(
      "static ::$proto_ns$::internal::once_flag $desc_table$_once;\n"
      "const ::$proto_ns$::internal::DescriptorTable $desc_table$ = {\n"
      "  false, $1$, $2$, $3$, \"$filename$\", \n"
      "  &$desc_table$_once, $4$, $5$, $6$,\n"
      "  schemas, file_default_instances, $tablename$::offsets,\n"
      "  $7$, $file_level_enum_descriptors$, "
      "$file_level_service_descriptors$,\n"
      "};\n"
      // This function exists to be marked as weak.
      // It can significantly speed up compilation by breaking up LLVM's SCC in
      // the .pb.cc translation units. Large translation units see a reduction
      // of more than 35% of walltime for optimized builds.
      // Without the weak attribute all the messages in the file, including all
      // the vtables and everything they use become part of the same SCC through
      // a cycle like:
      // GetMetadata -> descriptor table -> default instances ->
      //   vtables -> GetMetadata
      // By adding a weak function here we break the connection from the
      // individual vtables back into the descriptor table.
      "PROTOBUF_ATTRIBUTE_WEAK const ::$proto_ns$::internal::DescriptorTable* "
      "$desc_table$_getter() {\n"
      "  return &$desc_table$;\n"
      "}\n"
      "\n",
      eager ? "true" : "false", file_data.size(), protodef_name,
      num_deps == 0 ? "nullptr" : variables_["desc_table"] + "_deps", num_deps,
      message_generators_.size(),
      message_generators_.empty() ? "nullptr"
                                  : variables_["file_level_metadata"]);
}

table的注册

既然源文件中有序列化之后的字符串，就可以在运行时把这个字符串反序列化成一个描述实例。

(gdb) bt
#0  google::protobuf::EncodedDescriptorDatabase::Add (this=0x61dc90, encoded_file_descriptor=0x7ffff7ab6e20 <descriptor_table_protodef_google_2fprotobuf_2fany_2eproto>, size=212)
    at google/protobuf/descriptor_database.cc:556
#1  0x00007ffff78f3486 in google::protobuf::DescriptorPool::InternalAddGeneratedFile (encoded_file_descriptor=0x7ffff7ab6e20 <descriptor_table_protodef_google_2fprotobuf_2fany_2eproto>,
    size=212) at google/protobuf/descriptor.cc:1357
#2  0x00007ffff79adc27 in google::protobuf::(anonymous namespace)::AddDescriptorsImpl (table=0x7ffff7dd8b80 <descriptor_table_google_2fprotobuf_2fany_2eproto>)
    at google/protobuf/generated_message_reflection.cc:2767
#3  0x00007ffff79adc60 in google::protobuf::(anonymous namespace)::AddDescriptors (table=0x7ffff7dd8b80 <descriptor_table_google_2fprotobuf_2fany_2eproto>)
    at google/protobuf/generated_message_reflection.cc:2778
#4  0x00007ffff79add3c in google::protobuf::internal::AddDescriptorsRunner::AddDescriptorsRunner (this=0x7ffff7dda284 <dynamic_init_dummy_google_2fprotobuf_2fany_2eproto>,
    table=0x7ffff7dd8b80 <descriptor_table_google_2fprotobuf_2fany_2eproto>) at google/protobuf/generated_message_reflection.cc:2813
#5  0x00007ffff78e885e in __static_initialization_and_destruction_0 (__initialize_p=1, __priority=102) at google/protobuf/any.pb.cc:76
#6  0x00007ffff78e8874 in _GLOBAL__sub_I.00102_any.pb.cc(void) () at ./google/protobuf/port_undef.inc:128
#7  0x00007ffff7dea4c3 in _dl_init_internal () from /lib64/ld-linux-x86-64.so.2
#8  0x00007ffff7ddc1aa in _dl_start_user () from /lib64/ld-linux-x86-64.so.2
#9  0x0000000000000001 in ?? ()
#10 0x00007fffffffe5c5 in ?? ()
#11 0x0000000000000000 in ?? ()
(gdb)

在生成的源代码中有一个全局变量

  // Force running AddDescriptors() at dynamic initialization time.
  PROTOBUF_ATTRIBUTE_INIT_PRIORITY static ::PROTOBUF_NAMESPACE_ID::internal::AddDescriptorsRunner dynamic_init_dummy_tsecer_2eproto(&descriptor_table_tsecer_2eproto);

会触发internal::AddDescriptorsRunner构造函数，进而触发下面的流程，并调用file.ParseFromArray来解析这个字符串来还原原始的proto文件内容。

/// @file: protobuf\src\google\protobuf\generated_message_reflection.cc
void AssignDescriptorsImpl(const DescriptorTable* table, bool eager) {
///...
  // Register the descriptor of this file.
  DescriptorPool::InternalAddGeneratedFile(table->descriptor, table->size);
  MessageFactory::InternalRegisterGeneratedFile(table);
}

/// @file: google/protobuf/descriptor_database.cc:556
bool EncodedDescriptorDatabase::Add(const void* encoded_file_descriptor,
                                    int size) {
  FileDescriptorProto file;
  if (file.ParseFromArray(encoded_file_descriptor, size)) {
    return index_->AddFile(file, std::make_pair(encoded_file_descriptor, size));
  } else {
    GOOGLE_LOG(ERROR) << "Invalid file descriptor data passed to "
                  "EncodedDescriptorDatabase::Add().";
    return false;
  }
}

pb内部Descriptor的proto

在pb内部，把proto文件本身的message也定义成了proto文件。要注意的是proto文件不是通过这个描述自动解析的，这里只是通过定义生成了指定格式的容器和对应的操作接口，另外，由于使用proto描述，所以也自动拥有了proto的序列化、反序列化两个重要的基础功能。

// @file: protobuf\src\google\protobufdescriptor.proto
// Describes a field within a message.
// Describes a field within a message.
message FieldDescriptorProto {
  enum Type {
    // 0 is reserved for errors.
    // Order is weird for historical reasons.
    TYPE_DOUBLE = 1;
    TYPE_FLOAT = 2;
    // Not ZigZag encoded.  Negative numbers take 10 bytes.  Use TYPE_SINT64 if
    // negative values are likely.
    TYPE_INT64 = 3;
    TYPE_UINT64 = 4;
    // Not ZigZag encoded.  Negative numbers take 10 bytes.  Use TYPE_SINT32 if
    // negative values are likely.
    TYPE_INT32 = 5;
    TYPE_FIXED64 = 6;
    TYPE_FIXED32 = 7;
    TYPE_BOOL = 8;
    TYPE_STRING = 9;
    // Tag-delimited aggregate.
    // Group type is deprecated and not supported in proto3. However, Proto3
    // implementations should still be able to parse the group wire format and
    // treat group fields as unknown fields.
    TYPE_GROUP = 10;
    TYPE_MESSAGE = 11;  // Length-delimited aggregate.

    // New in version 2.
    TYPE_BYTES = 12;
    TYPE_UINT32 = 13;
    TYPE_ENUM = 14;
    TYPE_SFIXED32 = 15;
    TYPE_SFIXED64 = 16;
    TYPE_SINT32 = 17;  // Uses ZigZag encoding.
    TYPE_SINT64 = 18;  // Uses ZigZag encoding.
  }

  enum Label {
    // 0 is reserved for errors
    LABEL_OPTIONAL = 1;
    LABEL_REQUIRED = 2;
    LABEL_REPEATED = 3;
  }

  optional string name = 1;
  optional int32 number = 3;
  optional Label label = 4;

  // If type_name is set, this need not be set.  If both this and type_name
  // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP.
  optional Type type = 5;

字符串可视化

这里为什么使用8进制而没有使用更容易阅读的16进制呢？从代码上猜测：如果是6进制就可以直接把数值操作(除法或者取模)的结果加上'0'来获得一个；如果使用16进制，这个结果可能要经过一个表格转换之类的(因为 '0' + 10 != 'A')。

/// @file: protobuf\src\google\protobuf\stubs\strutil.cc
// ----------------------------------------------------------------------
// Escapes 'src' using C-style escape sequences, and appends the escaped string
// to 'dest'. This version is faster than calling CEscapeInternal as it computes
// the required space using a lookup table, and also does not do any special
// handling for Hex or UTF-8 characters.
// ----------------------------------------------------------------------
void CEscapeAndAppend(StringPiece src, std::string *dest) {
  size_t escaped_len = CEscapedLength(src);
  if (escaped_len == src.size()) {
    dest->append(src.data(), src.size());
    return;
  }

  size_t cur_dest_len = dest->size();
  dest->resize(cur_dest_len + escaped_len);
  char* append_ptr = &(*dest)[cur_dest_len];

  for (StringPiece::size_type i = 0; i < src.size(); ++i) {
    unsigned char c = static_cast<unsigned char>(src[i]);
    switch (c) {
      case '\n': *append_ptr++ = '\\'; *append_ptr++ = 'n'; break;
      case '\r': *append_ptr++ = '\\'; *append_ptr++ = 'r'; break;
      case '\t': *append_ptr++ = '\\'; *append_ptr++ = 't'; break;
      case '\"': *append_ptr++ = '\\'; *append_ptr++ = '\"'; break;
      case '\'': *append_ptr++ = '\\'; *append_ptr++ = '\''; break;
      case '\\': *append_ptr++ = '\\'; *append_ptr++ = '\\'; break;
      default:
        if (!isprint(c)) {
          *append_ptr++ = '\\';
          *append_ptr++ = '0' + c / 64;
          *append_ptr++ = '0' + (c % 64) / 8;
          *append_ptr++ = '0' + c % 8;
        } else {
          *append_ptr++ = c;
        }
        break;
    }
  }
}

栗子

描述文件

tsecer@harry: cat  tsecer.proto
syntax = "proto3";

message tsecer {
bytes harry = 1;
int32   fry   = 3;
};

tsecer@harry:

左括弧

在字符串描述中，经常会遇到醒目的左括弧字符，它是从哪里来？

其实这个是有前面

optional Type type = 5;

这个字段生成的：

由于'('字符的十六进制内码是0x28，对应的二进制是0b00101000，结合protobuf的编码格式。最低3个bit值为0，表示类型是一个VARINT；剩余内容为5，对应这里type字段的tag值为5。这个结果刚好是左括号而已。在左括弧之后，就是type的具体值了。在开始的例子中，输出中有两个左括弧，分别是"(\014"和"(\005"，括弧之后的八进制转换为10进制之后分别为15和5。

const char descriptor_table_protodef_tsecer_2eproto[] =
  "\n\014tsecer.proto\"$\n\006tsecer\022\r\n\005harry\030\001 \001(\014\022"
  "\013\n\003fry\030\003 \001(\005b\006proto3"
  ;

对应前面enum Type枚举类型，分别对应

    TYPE_INT32 = 5;
    TYPE_BYTES = 12;

对应，也和proto文件中的声明一致 。

descriptor.cc和descriptor.pb.cc

descriptor.pb.cc是protoc根据proto文件自动生成的，而descriptor.cc则是人工编写的代码。

///@file: protobuf\src\google\protobuf\descriptor.cc
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc.  All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:

descriptor.pb.cc只是作为一个解析之后的容器，它的数据也是有编译器在完成proto语法分析之后添加的。

在descriptor.cc文件中(人工)实现了DebugString接口，所以descriptor经过DebugString输出的格式跟常规Message输出的格式并不一样。

tsecer@harry: cat main.cpp
#include <iostream>
#include <fstream>
#include <string>
#include "tsecer.pb.h"
#include <google/protobuf/util/json_util.h>

using namespace std;

int main(int argc, char* argv[]) {
    char str[] = "\00\01\02\03HelloWorld";
tsecer msg;
auto desc = msg.GetDescriptor();
printf("%s\n", desc->DebugString ().c_str());
msg.set_harry(std::string(str, sizeof(str)));
printf("%s\n", msg.DebugString().c_str());
return 0;
}

tsecer@harry: ./a.out
message tsecer {
  bytes harry = 1;
  int32 fry = 3;
}

harry: "\000\001\002\003HelloWorld\000"

tsecer@harry: