Use C size for serialization/deserialization (#208)

This adds a new method in ByteRepr called byte_size. It is used to
describe the C/C++ byte size of the object and it's used for
serializatoin/deserialization through from/to_bytes.

In refcount, Rust structs have a different size than the C/C++ structs
because each field is wrapped in `Value<>`. For example:

```cpp
struct S {
  int32_t a;
  int32_t b;
}
sizeof(S) == 8
```

becomes

```rs
struct S {
  a: Value<i32>,
  b: Value<i32>
}
size_of(S) == 16 // one Rc is 8
```

This difference in size breaks ByteRepr for structs. To do correct
serialization we need the C/C++ size. Codegen embeds this into the
generated code using the ByteRepr::byte_size method.

Then, when a reinterpreted pointer is created,
ReinterpretedView::elem_byte_size saves ByteRepr::byte_size. This is an
optimization so that the ByteRepr trait is not exposed to all Ptr
methods, even into methods that don't have to do with reinterpreted
pointers.

The goal of this is to support structs inside unions. Structs, as any
other type that takes part in an union, is a reinterpreted view into the
byte buffer of the union.

Author: lucic71

cpp2rust/converter/converter_lib.cpp

@@ -200,6 +200,43 @@ bool IsMut(clang::QualType qual_type) {
            qual_type->getPointeeType().isConstQualified());
 }
 
+bool TypeImplementsByteRepr(clang::QualType qt) {
+  if (qt->isIntegerType() || qt->isFloatingType() || qt->isEnumeralType()) {
+    return true;
+  }
+  if (const auto *arr = qt->getAsArrayTypeUnsafe()) {
+    return TypeImplementsByteRepr(arr->getElementType());
+  }
+  if (const auto *rd = qt->getAsRecordDecl()) {
+    if (rd->getASTContext().getSourceManager().isInSystemHeader(
+            rd->getLocation())) {
+      return false;
+    }
+    if (rd->isUnion()) {
+      return false;
+    }
+    for (const auto *field : rd->fields()) {
+      if (!TypeImplementsByteRepr(field->getType())) {
+        return false;
+      }
+    }
+    return true;
+  }
+  return false;
+}
+
+bool RustSizeDivergesFromC(clang::QualType qt) {
+  qt = qt.getCanonicalType();
+  // Records have Rc<RefCell<>> fields that diverge from the C size
+  if (qt->isRecordType()) {
+    return true;
+  }
+  if (auto *arr = qt->getAsArrayTypeUnsafe()) {
+    return RustSizeDivergesFromC(arr->getElementType());
+  }
+  return false;
+}
+
 bool IsMutatingCall(const clang::CallExpr *expr) {
   if (auto *callee = expr->getDirectCallee()) {
     if (auto *method = clang::dyn_cast<clang::CXXMethodDecl>(callee)) {

cpp2rust/converter/converter_lib.h

@@ -52,6 +52,10 @@ bool IsUnsignedArithOp(const clang::BinaryOperator *expr);
 
 bool IsMut(clang::QualType qual_type);
 
+bool TypeImplementsByteRepr(clang::QualType qt);
+
+bool RustSizeDivergesFromC(clang::QualType qt);
+
 bool IsMutatingCall(const clang::CallExpr *expr);
 
 bool IsOverloadedFunction(const clang::FunctionDecl *decl);

cpp2rust/converter/models/converter_refcount.cpp

@@ -545,28 +545,10 @@ void ConverterRefCount::AddDropTrait(const clang::CXXRecordDecl *decl) {
   StrCat('}');
 }
 
-static bool recordImplementsByteRepr(const clang::RecordDecl *decl) {
-  if (decl->isUnion()) {
-    return false;
-  }
-
-  // ByteRepr is only supported for user-defined structs that contain ByteRepr
-  // fields.
-  for (auto *f : decl->fields()) {
-    auto qt = f->getType();
-    if (!qt->isIntegerType() && !qt->isFloatingType() &&
-        !qt->isEnumeralType()) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
 void ConverterRefCount::AddByteReprTrait(const clang::RecordDecl *decl) {
   auto struct_name = GetRecordName(decl);
 
-  if (!recordImplementsByteRepr(decl)) {
+  if (!TypeImplementsByteRepr(ctx_.getCanonicalTagType(decl))) {
     StrCat(std::format("impl ByteRepr for {}", struct_name));
     PushBrace brace(*this);
     return;
@@ -577,6 +559,9 @@ void ConverterRefCount::AddByteReprTrait(const clang::RecordDecl *decl) {
 
   const auto &layout = ctx_.getASTRecordLayout(decl);
 
+  StrCat(std::format("fn byte_size() -> usize {{ {} }}",
+                     ctx_.getTypeSize(ctx_.getCanonicalTagType(decl)) / 8));
+
   StrCat("fn to_bytes(&self, buf: &mut [u8])");
   {
     PushBrace fn_brace(*this);
@@ -600,9 +585,12 @@ void ConverterRefCount::AddByteReprTrait(const clang::RecordDecl *decl) {
     for (auto *field : decl->fields()) {
       auto byte_off = layout.getFieldOffset(idx) / 8;
       auto byte_size = ctx_.getTypeSize(field->getType()) / 8;
+      PushConversionKind push(*this, ConversionKind::FullRefCount);
+      std::string storage_ty = ToString(field->getType());
+      Unwrap(storage_ty, "Value<", ">");
       StrCat(std::format(
           "{}: Rc::new(RefCell::new(<{}>::from_bytes(&buf[{}..{}]))),",
-          GetNamedDeclAsString(field), Mapper::Map(field->getType()), byte_off,
+          GetNamedDeclAsString(field), storage_ty, byte_off,
           byte_off + byte_size));
       ++idx;
     }
@@ -1322,6 +1310,20 @@ bool ConverterRefCount::VisitExplicitCastExpr(clang::ExplicitCastExpr *expr) {
   }
 }
 
+bool ConverterRefCount::VisitUnaryExprOrTypeTraitExpr(
+    clang::UnaryExprOrTypeTraitExpr *expr) {
+  if (expr->getKind() == clang::UnaryExprOrTypeTrait::UETT_SizeOf) {
+    auto arg_type = expr->isArgumentType() ? expr->getArgumentType()
+                                           : expr->getArgumentExpr()->getType();
+    if (RustSizeDivergesFromC(arg_type)) {
+      StrCat(std::format("{}usize", ctx_.getTypeSize(arg_type) / 8));
+      computed_expr_type_ = ComputedExprType::FreshValue;
+      return false;
+    }
+  }
+  return Converter::VisitUnaryExprOrTypeTraitExpr(expr);
+}
+
 bool ConverterRefCount::VisitStmtExpr(clang::StmtExpr *expr) {
   PushConversionKind push(*this, ConversionKind::FullRefCount);
   return Converter::VisitStmtExpr(expr);

cpp2rust/converter/models/converter_refcount.h

@@ -43,6 +43,9 @@ class ConverterRefCount final : public Converter {
 
   void AddByteReprTrait(const clang::EnumDecl *decl) override;
 
+  bool
+  VisitUnaryExprOrTypeTraitExpr(clang::UnaryExprOrTypeTraitExpr *expr) override;
+
   void AddDefaultTrait(const clang::RecordDecl *decl) override;
 
   void AddDefaultTraitForUnion(const clang::RecordDecl *decl) override;

libcc2rs/src/rc.rs

@@ -16,6 +16,13 @@ use crate::reinterpret::{ByteRepr, OriginalAlloc, SingleOriginalAlloc, SliceOrig
 
 pub type Value<T> = Rc<RefCell<T>>;
 
+struct ReinterpretedView {
+    // Pointer to the source of reinterpret
+    alloc: Rc<dyn OriginalAlloc>,
+    // C++ size of the reinterpreted view
+    elem_byte_size: usize,
+}
+
 #[derive(Default)]
 enum PtrKind<T> {
     #[default]
@@ -25,7 +32,7 @@ enum PtrKind<T> {
     HeapSingle(Weak<RefCell<T>>),
     HeapArray(Weak<RefCell<Box<[T]>>>),
     Vec(Weak<RefCell<Vec<T>>>),
-    Reinterpreted(Rc<dyn OriginalAlloc>),
+    Reinterpreted(Rc<ReinterpretedView>),
 }
 
 pub enum StrongPtr<T> {
@@ -59,7 +66,7 @@ impl<T: ByteRepr> StrongPtr<T> {
                 cell,
             } => {
                 // Read-through: always re-read from the original allocation.
-                let mut buf = vec![0u8; std::mem::size_of::<T>()];
+                let mut buf = vec![0u8; T::byte_size()];
                 alloc.read_bytes(*byte_offset, &mut buf);
                 *cell.borrow_mut() = Some(T::from_bytes(&buf));
                 Ref::map(cell.borrow(), |opt| opt.as_ref().unwrap())
@@ -78,7 +85,7 @@ impl<T> fmt::Debug for PtrKind<T> {
             PtrKind::StackArray(w) => write!(f, "StackArray({:?})", w.as_ptr()),
             PtrKind::HeapArray(w) => write!(f, "HeapArray({:?})", w.as_ptr()),
             PtrKind::Reinterpreted(data) => {
-                write!(f, "Reinterpreted(0x{:x})", data.address())
+                write!(f, "Reinterpreted(0x{:x})", data.alloc.address())
             }
         }
     }
@@ -105,7 +112,7 @@ impl<T> PtrKind<T> {
             PtrKind::StackSingle(w) | PtrKind::HeapSingle(w) => w.as_ptr() as usize,
             PtrKind::Vec(w) => w.as_ptr() as usize,
             PtrKind::StackArray(w) | PtrKind::HeapArray(w) => w.as_ptr() as usize,
-            PtrKind::Reinterpreted(data) => data.address(),
+            PtrKind::Reinterpreted(data) => data.alloc.address(),
         }
     }
 }
@@ -251,40 +258,40 @@ impl<T> Ptr<T> {
     #[inline]
     fn elem_step(&self) -> usize {
         match &self.kind {
-            PtrKind::Reinterpreted(_) => std::mem::size_of::<T>(),
+            PtrKind::Reinterpreted(data) => data.elem_byte_size,
             _ => 1,
         }
     }
 
     #[inline]
     pub fn len(&self) -> usize {
-        match self.kind {
+        match &self.kind {
             PtrKind::Null => 0,
             PtrKind::StackSingle(_) | PtrKind::HeapSingle(_) => 1,
-            PtrKind::Vec(ref weak) => weak.upgrade().expect("ub: dangling pointer").borrow().len(),
-            PtrKind::StackArray(ref weak) | PtrKind::HeapArray(ref weak) => {
+            PtrKind::Vec(weak) => weak.upgrade().expect("ub: dangling pointer").borrow().len(),
+            PtrKind::StackArray(weak) | PtrKind::HeapArray(weak) => {
                 weak.upgrade().expect("ub: dangling pointer").borrow().len()
             }
-            PtrKind::Reinterpreted(ref data) => data.total_byte_len() / std::mem::size_of::<T>(),
+            PtrKind::Reinterpreted(data) => data.alloc.total_byte_len() / data.elem_byte_size,
         }
     }
 
     #[inline]
     pub fn is_empty(&self) -> bool {
-        match self.kind {
+        match &self.kind {
             PtrKind::Null => true,
             PtrKind::StackSingle(_) | PtrKind::HeapSingle(_) => false,
-            PtrKind::Vec(ref weak) => weak
+            PtrKind::Vec(weak) => weak
                 .upgrade()
                 .expect("ub: dangling pointer")
                 .borrow()
                 .is_empty(),
-            PtrKind::StackArray(ref weak) | PtrKind::HeapArray(ref weak) => weak
+            PtrKind::StackArray(weak) | PtrKind::HeapArray(weak) => weak
                 .upgrade()
                 .expect("ub: dangling pointer")
                 .borrow()
                 .is_empty(),
-            PtrKind::Reinterpreted(ref data) => self.offset >= data.total_byte_len(),
+            PtrKind::Reinterpreted(data) => self.offset >= data.alloc.total_byte_len(),
         }
     }
 
@@ -340,7 +347,7 @@ impl<T> Ptr<T> {
                 offset: self.offset,
             },
             PtrKind::Reinterpreted(data) => StrongPtr::Reinterpreted {
-                alloc: Rc::clone(data),
+                alloc: Rc::clone(&data.alloc),
                 byte_offset: self.offset,
                 cell: RefCell::new(None),
             },
@@ -366,9 +373,9 @@ impl<T> Ptr<T> {
                 rc.borrow_mut()[self.offset] = value;
             }
             PtrKind::Reinterpreted(data) => {
-                let mut buf = vec![0u8; std::mem::size_of::<T>()];
+                let mut buf = vec![0u8; T::byte_size()];
                 value.to_bytes(&mut buf);
-                data.write_bytes(self.offset, &buf);
+                data.alloc.write_bytes(self.offset, &buf);
             }
         }
     }
@@ -391,30 +398,34 @@ impl<T> Ptr<T> {
             return self_any.downcast_ref::<Ptr<U>>().unwrap().clone();
         }
 
-        if std::mem::size_of::<U>() == 0 {
+        if U::byte_size() == 0 {
             panic!("cannot reinterpret_cast to zero-sized type");
         }
 
+        let src_byte_off = self.offset.wrapping_mul(T::byte_size());
         let (alloc, abs_byte_off): (Rc<dyn OriginalAlloc>, usize) = match &self.kind {
             PtrKind::Null => return Ptr::null(),
             PtrKind::StackSingle(weak) | PtrKind::HeapSingle(weak) => (
                 Rc::new(SingleOriginalAlloc { weak: weak.clone() }),
-                self.byte_offset(),
+                src_byte_off,
             ),
             PtrKind::Vec(weak) => (
                 Rc::new(SliceOriginalAlloc { weak: weak.clone() }),
-                self.byte_offset(),
+                src_byte_off,
             ),
             PtrKind::StackArray(weak) | PtrKind::HeapArray(weak) => (
                 Rc::new(SliceOriginalAlloc { weak: weak.clone() }),
-                self.byte_offset(),
+                src_byte_off,
             ),
-            PtrKind::Reinterpreted(data) => (Rc::clone(data), self.offset),
+            PtrKind::Reinterpreted(data) => (Rc::clone(&data.alloc), self.offset),
         };
 
         Ptr {
             offset: abs_byte_off,
-            kind: PtrKind::Reinterpreted(alloc),
+            kind: PtrKind::Reinterpreted(Rc::new(ReinterpretedView {
+                alloc,
+                elem_byte_size: U::byte_size(),
+            })),
         }
     }
 }
@@ -442,12 +453,12 @@ impl<T> Ptr<T> {
                 f(&mut borrow[self.offset])
             }
             PtrKind::Reinterpreted(data) => {
-                let mut buf = vec![0u8; std::mem::size_of::<T>()];
-                data.read_bytes(self.offset, &mut buf);
+                let mut buf = vec![0u8; T::byte_size()];
+                data.alloc.read_bytes(self.offset, &mut buf);
                 let mut val = T::from_bytes(&buf);
                 let ret = f(&mut val);
                 val.to_bytes(&mut buf);
-                data.write_bytes(self.offset, &buf);
+                data.alloc.write_bytes(self.offset, &buf);
                 ret
             }
         }
@@ -475,8 +486,8 @@ impl<T> Ptr<T> {
                 f(&borrow[self.offset])
             }
             PtrKind::Reinterpreted(data) => {
-                let mut buf = vec![0u8; std::mem::size_of::<T>()];
-                data.read_bytes(self.offset, &mut buf);
+                let mut buf = vec![0u8; T::byte_size()];
+                data.alloc.read_bytes(self.offset, &mut buf);
                 let val = T::from_bytes(&buf);
                 f(&val)
             }
@@ -502,8 +513,8 @@ impl<T: Clone + ByteRepr> Ptr<T> {
                 weak.upgrade().expect("ub: dangling pointer").borrow()[self.offset].clone()
             }
             PtrKind::Reinterpreted(ref data) => {
-                let mut buf = vec![0u8; std::mem::size_of::<T>()];
-                data.read_bytes(self.offset, &mut buf);
+                let mut buf = vec![0u8; T::byte_size()];
+                data.alloc.read_bytes(self.offset, &mut buf);
                 T::from_bytes(&buf)
             }
         }
@@ -535,7 +546,7 @@ impl<T: std::cmp::Ord> Ptr<T> {
                 let strong = weak.upgrade().expect("ub: dangling pointer");
                 (*strong.borrow_mut())[self.get_offset()..last].sort();
             }
-            PtrKind::Reinterpreted(..) => {
+            PtrKind::Reinterpreted(_) => {
                 panic!("sorting not supported for reinterpreted pointers")
             }
         }
@@ -580,7 +591,7 @@ impl<T: Clone> Ptr<T> {
                 let mut borrow = strong.borrow_mut();
                 sort(&mut borrow, self.get_offset(), last, &mut cmp);
             }
-            PtrKind::Reinterpreted(..) => {
+            PtrKind::Reinterpreted(_) => {
                 panic!("sorting not supported for reinterpreted pointers")
             }
         }
@@ -856,7 +867,7 @@ impl<T> ToOwnedOption<T, T> for Ptr<T> {
             }
             PtrKind::Vec(_) => panic!("Can't own a vector"),
             PtrKind::HeapArray(_) => panic!("Can't own an array variable as single"),
-            PtrKind::Reinterpreted(..) => panic!("Can't own a reinterpreted pointer"),
+            PtrKind::Reinterpreted(_) => panic!("Can't own a reinterpreted pointer"),
         }
     }
 }
@@ -882,7 +893,7 @@ impl<T> ToOwnedOption<T, Box<[T]>> for Ptr<T> {
             }
             PtrKind::Vec(_) => panic!("Can't own a vector"),
             PtrKind::HeapSingle(_) => panic!("Can't own a single variable as an array"),
-            PtrKind::Reinterpreted(..) => panic!("Can't own a reinterpreted pointer"),
+            PtrKind::Reinterpreted(_) => panic!("Can't own a reinterpreted pointer"),
         }
     }
 }
@@ -898,7 +909,7 @@ impl<T> fmt::Debug for Ptr<T> {
                 (Weak::as_ptr(w) as usize).wrapping_add(self.byte_offset())
             }
             PtrKind::Vec(w) => (Weak::as_ptr(w) as usize).wrapping_add(self.byte_offset()),
-            PtrKind::Reinterpreted(data) => data.address().wrapping_add(self.byte_offset()),
+            PtrKind::Reinterpreted(data) => data.alloc.address().wrapping_add(self.byte_offset()),
         };
         write!(f, "0x{:x}", addr)
     }
@@ -996,7 +1007,7 @@ impl Ptr<u8> {
             }
             PtrKind::Reinterpreted(ref data) => {
                 let mut buf = vec![0u8; end.wrapping_sub(start)];
-                data.read_bytes(start, &mut buf);
+                data.alloc.read_bytes(start, &mut buf);
                 buf
             }
         }