Fix TcfCaV1 (Canada) OptimizedRange encoding to use Fibonacci, with backwards-compatible decoding

iabgpp-encoder/src/main/java/com/iab/gpp/encoder/datatype/EncodableArrayOfOptimizedFibonacciRanges.java

@@ -0,0 +1,118 @@
+package com.iab.gpp.encoder.datatype;
+
+import java.util.ArrayList;
+import java.util.List;
+import com.iab.gpp.encoder.datatype.encoder.FixedIntegerEncoder;
+import com.iab.gpp.encoder.datatype.encoder.OptimizedFibonacciRangeEncoder;
+import com.iab.gpp.encoder.error.DecodingException;
+import com.iab.gpp.encoder.error.EncodingException;
+
+/**
+ * Encodes an N-ArrayOfRanges(X,Y) field where each record's ids are encoded as an OptimizedRange
+ * (Fibonacci coded), per the GPP Consent String Specification. This is the counterpart to
+ * {@link EncodableArrayOfFixedIntegerRanges}, which encodes ids using fixed-integer ranges for
+ * downward compatibility (e.g. TCF EU).
+ */
+public class EncodableArrayOfOptimizedFibonacciRanges extends AbstractEncodableBitStringDataType<List<RangeEntry>> {
+
+  private int keyBitStringLength;
+  private int typeBitStringLength;
+
+  protected EncodableArrayOfOptimizedFibonacciRanges(int keyBitStringLength, int typeBitStringLength) {
+    super(true);
+    this.keyBitStringLength = keyBitStringLength;
+    this.typeBitStringLength = typeBitStringLength;
+  }
+
+  public EncodableArrayOfOptimizedFibonacciRanges(int keyBitStringLength, int typeBitStringLength,
+      List<RangeEntry> value) {
+    super(true);
+    this.keyBitStringLength = keyBitStringLength;
+    this.typeBitStringLength = typeBitStringLength;
+    setValue(value);
+  }
+
+  public EncodableArrayOfOptimizedFibonacciRanges(int keyBitStringLength, int typeBitStringLength,
+      List<RangeEntry> value, boolean hardFailIfMissing) {
+    super(hardFailIfMissing);
+    this.keyBitStringLength = keyBitStringLength;
+    this.typeBitStringLength = typeBitStringLength;
+    setValue(value);
+  }
+
+  @Override
+  public String encode() {
+    try {
+      List<RangeEntry> entries = this.value;
+
+      StringBuilder sb = new StringBuilder();
+      sb.append(FixedIntegerEncoder.encode(entries.size(), 12));
+      for (RangeEntry entry : entries) {
+        sb.append(FixedIntegerEncoder.encode(entry.getKey(), keyBitStringLength))
+            .append(FixedIntegerEncoder.encode(entry.getType(), typeBitStringLength))
+            .append(OptimizedFibonacciRangeEncoder.encode(entry.getIds()));
+      }
+
+      return sb.toString();
+    } catch (Exception e) {
+      throw new EncodingException(e);
+    }
+  }
+
+  @Override
+  public void decode(String bitString) {
+    try {
+      List<RangeEntry> entries = new ArrayList<>();
+
+      int size = FixedIntegerEncoder.decode(bitString.substring(0, 12));
+      int index = 12;
+      for (int i = 0; i < size; i++) {
+        int key = FixedIntegerEncoder.decode(bitString.substring(index, index + keyBitStringLength));
+        index += keyBitStringLength;
+
+        int type = FixedIntegerEncoder.decode(bitString.substring(index, index + typeBitStringLength));
+        index += typeBitStringLength;
+
+        String substring = new EncodableOptimizedFibonacciRange(new ArrayList<>()).substring(bitString, index);
+        List<Integer> ids = OptimizedFibonacciRangeEncoder.decode(substring);
+        index += substring.length();
+
+        entries.add(new RangeEntry(key, type, ids));
+      }
+
+      this.value = entries;
+    } catch (Exception e) {
+      throw new DecodingException(e);
+    }
+  }
+
+  @Override
+  public String substring(String bitString, int fromIndex) throws SubstringException {
+    try {
+      StringBuilder sb = new StringBuilder();
+      sb.append(bitString.substring(fromIndex, fromIndex + 12));
+
+      int size = FixedIntegerEncoder.decode(sb.toString());
+
+      int index = fromIndex + sb.length();
+      for (int i = 0; i < size; i++) {
+        String keySubstring = bitString.substring(index, index + keyBitStringLength);
+        index += keySubstring.length();
+        sb.append(keySubstring);
+
+        String typeSubstring = bitString.substring(index, index + typeBitStringLength);
+        index += typeSubstring.length();
+        sb.append(typeSubstring);
+
+        String rangeSubstring = new EncodableOptimizedFibonacciRange(new ArrayList<>()).substring(bitString, index);
+        index += rangeSubstring.length();
+        sb.append(rangeSubstring);
+      }
+
+      return sb.toString();
+    } catch (Exception e) {
+      throw new SubstringException(e);
+    }
+  }
+
+}

iabgpp-encoder/src/main/java/com/iab/gpp/encoder/segment/TcfCaV1CoreSegment.java

@@ -8,11 +8,13 @@
 import com.iab.gpp.encoder.base64.CompressedBase64UrlEncoder;
 import com.iab.gpp.encoder.bitstring.BitStringEncoder;
 import com.iab.gpp.encoder.datatype.EncodableArrayOfFixedIntegerRanges;
+import com.iab.gpp.encoder.datatype.EncodableArrayOfOptimizedFibonacciRanges;
 import com.iab.gpp.encoder.datatype.EncodableBoolean;
 import com.iab.gpp.encoder.datatype.EncodableDatetime;
 import com.iab.gpp.encoder.datatype.EncodableFixedBitfield;
 import com.iab.gpp.encoder.datatype.EncodableFixedInteger;
 import com.iab.gpp.encoder.datatype.EncodableFixedString;
+import com.iab.gpp.encoder.datatype.EncodableOptimizedFibonacciRange;
 import com.iab.gpp.encoder.datatype.EncodableOptimizedFixedRange;
 import com.iab.gpp.encoder.error.DecodingException;
 import com.iab.gpp.encoder.field.EncodableBitStringFields;
@@ -37,11 +39,21 @@ public TcfCaV1CoreSegment(String encodedString) {
   public List<String> getFieldNames() {
     return TcfCaV1Field.TCFCAV1_CORE_SEGMENT_FIELD_NAMES;
   }
-  
+
   @Override
   protected EncodableBitStringFields initializeFields() {
+    return buildFields(false);
+  }
+
+  /**
+   * Builds the core field set. When {@code legacy} is true the OptimizedRange / N-ArrayOfRanges
+   * fields use the pre-fix fixed-integer encoders; otherwise they use the spec-compliant Fibonacci
+   * encoders. The legacy field set is only used to decode strings produced by the older encoder
+   * (see {@link #decodeSegment}).
+   */
+  private EncodableBitStringFields buildFields(boolean legacy) {
     ZonedDateTime date = ZonedDateTime.now();
-    
+
     EncodableBitStringFields fields = new EncodableBitStringFields();
     fields.put(TcfCaV1Field.VERSION, new EncodableFixedInteger(6, TcfCaV1.VERSION));
     fields.put(TcfCaV1Field.CREATED, new EncodableDatetime(date));
@@ -61,9 +73,17 @@ protected EncodableBitStringFields initializeFields() {
     fields.put(TcfCaV1Field.PURPOSES_IMPLIED_CONSENT,
         new EncodableFixedBitfield(Arrays.asList(false, false, false, false, false, false, false, false, false, false,
             false, false, false, false, false, false, false, false, false, false, false, false, false, false)));
-    fields.put(TcfCaV1Field.VENDOR_EXPRESS_CONSENT, new EncodableOptimizedFixedRange(new ArrayList<>()));
-    fields.put(TcfCaV1Field.VENDOR_IMPLIED_CONSENT, new EncodableOptimizedFixedRange(new ArrayList<>()));
-    fields.put(TcfCaV1Field.PUB_RESTRICTIONS, new EncodableArrayOfFixedIntegerRanges(6, 2, new ArrayList<>(), false));
+
+    if (legacy) {
+      fields.put(TcfCaV1Field.VENDOR_EXPRESS_CONSENT, new EncodableOptimizedFixedRange(new ArrayList<>()));
+      fields.put(TcfCaV1Field.VENDOR_IMPLIED_CONSENT, new EncodableOptimizedFixedRange(new ArrayList<>()));
+      fields.put(TcfCaV1Field.PUB_RESTRICTIONS, new EncodableArrayOfFixedIntegerRanges(6, 2, new ArrayList<>(), false));
+    } else {
+      fields.put(TcfCaV1Field.VENDOR_EXPRESS_CONSENT, new EncodableOptimizedFibonacciRange(new ArrayList<>()));
+      fields.put(TcfCaV1Field.VENDOR_IMPLIED_CONSENT, new EncodableOptimizedFibonacciRange(new ArrayList<>()));
+      fields.put(TcfCaV1Field.PUB_RESTRICTIONS,
+          new EncodableArrayOfOptimizedFibonacciRanges(6, 2, new ArrayList<>(), false));
+    }
     return fields;
   }
 
@@ -76,14 +96,52 @@ protected String encodeSegment(EncodableBitStringFields fields) {
 
   @Override
   protected void decodeSegment(String encodedString, EncodableBitStringFields fields) {
-    if(encodedString == null || encodedString.isEmpty()) {
+    if (encodedString == null || encodedString.isEmpty()) {
       this.fields.reset(fields);
     }
     try {
       String bitString = base64UrlEncoder.decode(encodedString);
+
+      // Prefer the spec-compliant (Fibonacci OptimizedRange) interpretation. If that doesn't
+      // round-trip back to the input, fall back to the legacy (fixed-range) interpretation used by
+      // the pre-fix encoder. This keeps older strings decodable; re-encoding always migrates them
+      // to the spec-compliant format because the fields decode into the Fibonacci datatypes.
+      if (tryDecode(bitString, fields, false)) {
+        return;
+      }
+      if (tryDecode(bitString, fields, true)) {
+        return;
+      }
+
+      // Neither interpretation round-trips cleanly; decode with the current interpretation as a
+      // best effort and surface any decoding error.
       bitStringEncoder.decode(bitString, getFieldNames(), fields);
     } catch (Exception e) {
       throw new DecodingException("Unable to decode TcfCaV1CoreSegment '" + encodedString + "'", e);
     }
   }
+
+  /**
+   * Attempts to decode {@code bitString} using either the current or legacy field set and verifies
+   * the result by re-encoding it: if the re-encoded bits are a prefix of the decoded bits (the tail
+   * being base64 padding), the interpretation produced the string. On success the decoded values
+   * are copied into {@code targetFields} (which always use the current encoders) so that any
+   * subsequent re-encode emits the spec-compliant format.
+   */
+  private boolean tryDecode(String bitString, EncodableBitStringFields targetFields, boolean legacy) {
+    try {
+      EncodableBitStringFields candidate = buildFields(legacy);
+      bitStringEncoder.decode(bitString, getFieldNames(), candidate);
+      String reEncoded = bitStringEncoder.encode(candidate, getFieldNames());
+      if (bitString.startsWith(reEncoded)) {
+        for (String fieldName : getFieldNames()) {
+          targetFields.get(fieldName).setValue(candidate.get(fieldName).getValue());
+        }
+        return true;
+      }
+    } catch (Exception e) {
+      // This interpretation does not apply; the caller will try the next one.
+    }
+    return false;
+  }
 }

iabgpp-encoder/src/main/java/com/iab/gpp/encoder/segment/TcfCaV1DisclosedVendorsSegment.java

@@ -6,6 +6,7 @@
 import com.iab.gpp.encoder.base64.CompressedBase64UrlEncoder;
 import com.iab.gpp.encoder.bitstring.BitStringEncoder;
 import com.iab.gpp.encoder.datatype.EncodableFixedInteger;
+import com.iab.gpp.encoder.datatype.EncodableOptimizedFibonacciRange;
 import com.iab.gpp.encoder.datatype.EncodableOptimizedFixedRange;
 import com.iab.gpp.encoder.error.DecodingException;
 import com.iab.gpp.encoder.field.EncodableBitStringFields;
@@ -32,9 +33,23 @@ public List<String> getFieldNames() {
 
   @Override
   protected EncodableBitStringFields initializeFields() {
+    return buildFields(false);
+  }
+
+  /**
+   * Builds the disclosed-vendors field set. When {@code legacy} is true the OptimizedRange field
+   * uses the pre-fix fixed-integer encoder; otherwise it uses the spec-compliant Fibonacci encoder.
+   * The legacy field set is only used to decode strings produced by the older encoder (see
+   * {@link #decodeSegment}).
+   */
+  private EncodableBitStringFields buildFields(boolean legacy) {
     EncodableBitStringFields fields = new EncodableBitStringFields();
     fields.put(TcfCaV1Field.DISCLOSED_VENDORS_SEGMENT_TYPE, new EncodableFixedInteger(3, 1));
-    fields.put(TcfCaV1Field.DISCLOSED_VENDORS, new EncodableOptimizedFixedRange(new ArrayList<>()));
+    if (legacy) {
+      fields.put(TcfCaV1Field.DISCLOSED_VENDORS, new EncodableOptimizedFixedRange(new ArrayList<>()));
+    } else {
+      fields.put(TcfCaV1Field.DISCLOSED_VENDORS, new EncodableOptimizedFibonacciRange(new ArrayList<>()));
+    }
     return fields;
   }
 
@@ -52,9 +67,44 @@ protected void decodeSegment(String encodedString, EncodableBitStringFields fiel
     }
     try {
       String bitString = base64UrlEncoder.decode(encodedString);
+
+      // Prefer the spec-compliant (Fibonacci OptimizedRange) interpretation, falling back to the
+      // legacy (fixed-range) interpretation used by the pre-fix encoder. Re-encoding always
+      // migrates to the spec-compliant format because the values decode into the Fibonacci datatype.
+      if (tryDecode(bitString, fields, false)) {
+        return;
+      }
+      if (tryDecode(bitString, fields, true)) {
+        return;
+      }
+
       bitStringEncoder.decode(bitString, getFieldNames(), fields);
     } catch (Exception e) {
       throw new DecodingException("Unable to decode TcfCaV1DisclosedVendorsSegment '" + encodedString + "'", e);
     }
   }
+
+  /**
+   * Attempts to decode {@code bitString} using either the current or legacy field set and verifies
+   * the result by re-encoding it: if the re-encoded bits are a prefix of the decoded bits (the tail
+   * being base64 padding), the interpretation produced the string. On success the decoded values
+   * are copied into {@code targetFields} (which always use the current encoders) so that any
+   * subsequent re-encode emits the spec-compliant format.
+   */
+  private boolean tryDecode(String bitString, EncodableBitStringFields targetFields, boolean legacy) {
+    try {
+      EncodableBitStringFields candidate = buildFields(legacy);
+      bitStringEncoder.decode(bitString, getFieldNames(), candidate);
+      String reEncoded = bitStringEncoder.encode(candidate, getFieldNames());
+      if (bitString.startsWith(reEncoded)) {
+        for (String fieldName : getFieldNames()) {
+          targetFields.get(fieldName).setValue(candidate.get(fieldName).getValue());
+        }
+        return true;
+      }
+    } catch (Exception e) {
+      // This interpretation does not apply; the caller will try the next one.
+    }
+    return false;
+  }
 }

iabgpp-encoder/src/test/java/com/iab/gpp/encoder/GppModelTest.java

@@ -373,7 +373,7 @@ public void testEncodeUspV1AndTcfEuV2AndTcfCaV1() {
 
     String gppString = gppModel.encode();
     Assertions.assertEquals(
-        "DBACOeA~CPSG_8APSG_8ANwAAAENAwCAAAAAAAAAAAAAAAAAAAAA.QAAA.IAAA~BPSG_8APSG_8AAyACAENGdCgf_gfgAfgfgBgABABAAABAB4AACACAAA.fHHHA4444ao~1YNN",
+        "DBACOeA~CPSG_8APSG_8ANwAAAENAwCAAAAAAAAAAAAAAAAAAAAA.QAAA.IAAA~BPSG_8APSG_8AAyACAENGdCgf_gfgAfgfgBhADVqxGAD0AILVgAA.fHHHA4444ao~1YNN",
         gppString);
 
     Assertions.assertEquals(4, gppString.split("~").length);
@@ -564,7 +564,7 @@ public void testDecodeUspv1AndTcfEuV2() {
   @Test
   public void testDecodeUspv1AndTcfEuV2AndTcfCaV1() {
     String gppString =
-        "DBACOeA~CPSG_8APSG_8ANwAAAENAwCAAAAAAAAAAAAAAAAAAAAA.QAAA.IAAA~BPSG_8APSG_8AAyACAENGdCgf_gfgAfgfgBgABABAAABAB4AACACAAA.fHHHA4444ao~1YNN";
+        "DBACOeA~CPSG_8APSG_8ANwAAAENAwCAAAAAAAAAAAAAAAAAAAAA.QAAA.IAAA~BPSG_8APSG_8AAyACAENGdCgf_gfgAfgfgBhADVqxGAD0AILVgAA.fHHHA4444ao~1YNN";
     GppModel gppModel = new GppModel(gppString);
 
     Assertions.assertEquals(Arrays.asList(2, 5, 6), gppModel.getSectionIds());