Add End to End Unit Testing for JSON De/Serialization (#79)

* update names of mock methods

* fix naming of json annotations for proper updates

* Updates to support proper sequence generation

* add printing of object and json outputs for manual review and update test description

* adds support for IdentitySDO

* add identity SDO testing support

* Add random label generator

* Add support for Campaign SDO

* remove tabs

* Add course of action, intrusion set tests + fixes to related SDOs + cleanup

* add Malware, ObservedData + Artifact + Fix for hashes pluralization

* remove old imports

* Fix misnamed method

* Add Autonomous System COO

* Add DirectoryCOO

* Add Domain Name COO

* Add Email Address COO

* removing inner Set<String> from headers to conform to spec, even though its a issue

The spec dictates this is a "Dictionary" field.  But a dictionary cannot have duplicate fields.  oasis-tcs/cti-stix2#138

* Add Email Message COO + MimePartType which is used in email message

* Fix bad business rule for email message

* add comment

* fix bad naming of jsonpropertydescription

* Add dupluralizeDictionary values for better method naming

* reorder methods to match the ordering in the spec

* Add support for File Coo

* Cleanup on COO Extension serializer and deserializer for better output

* Remove unneeded Annotations as the MimePartType only exists as a sub object and has not "typing"

* Add Valid annotation to getBodyMultipart so the MimePartTypeObj is validated against the EmailMessage validation

* cleanup + fix file coo issues

* add missing GenericValdiation extends on COO inner types

* Remove valid anotation as it will be handled by the GenericValidation interface of MimePartTypeObj

* convert list mocking to use fromStrings() method provided by mockneat

* remove old code for better Mimes dictionary

* adjust default range for sector generation in identites

* change custom props to generate with prepend

* fix regression with threat actor sdo

* Add IVP4 COO supporte

* Add IPV6 COO support

* Add Mac Address COO + MacAddress generation support on IVP4 and IPV6 COO

* fix types

* Add Mutex and NetworkTraffic COO

* Fix type in method name

* Add Process COO support

* Remove regex until better working pattern can be created

* Add Software COO

* Add URL coo

* fix vocab placement

* Add UserAccount COO and cleanup

* Add user account coo to Observed Object generator

* fix vocab annotation placement

* cleanup todo

* Add Windows Registry Key Coo support

* Fix network traffic annotations for range

* fix regex pattern annotaiton placement

* Cleanup to fix bugs

* Cleanup + Add depluralize dictionary support

* Add X509 cert COO support

* Add TheatActor, Report, Tool, Vulnerability SDO support plus cleanup

* Add alias/aliases dictionary for depluralization of methods

* Fix annotation for vocab on labels

* Fix business rule that was using incorrect methods (isEmpty vs isPresent)

* Add jsonassert

* Update attack pattern with use of objectmapper and json assert

* Update for using ObjectMapper for serialization/deserialization within specs

* cleanup imports

* remove unneeded imports from groovy.json

* Remove unneeded Style annotation

* Add custom properties support onto the Bundle object

* Add mock bundle support

* Remove old tests that are not needed

* remove old json files from tests

* refactor and add readme context for test resources

* refactor mockDatagenerator groovy trait location

* Add custom props support to Marking Objects and Granular Marking Objects

* fix javadoc

* Add Granular Marking, Object Marking, Statement and TLP support + updated all SDOs with CreatedByRef + Updated Report and Bundle to generate Object Markings

* remove markovs are they are way too slow!

* rename method for clearner usage

* Add Markings Definition support + JsontypeName support for TLP and Statement Markings

* refactor vocabularies into clearner packages

* update deps and add missing imports for refactor of vocabs packages

* Add Relationship and Sighting SROs support.  Added to Bundle and Report + Add individual Spec tests

Author: StephenOTT

pom.xml

@@ -153,6 +153,13 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.skyscreamer</groupId>
+            <artifactId>jsonassert</artifactId>
+            <version>1.5.0</version>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
     <build>

src/main/java/io/digitalstate/stix/bundle/BundleObject.java

@@ -7,6 +7,7 @@
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.digitalstate.stix.common.StixCustomProperties;
 import io.digitalstate.stix.helpers.StixSpecVersion;
 import io.digitalstate.stix.json.StixParsers;
 import io.digitalstate.stix.validation.GenericValidation;
@@ -34,7 +35,7 @@
 @Value.Style(typeImmutable = "Bundle", validationMethod = Value.Style.ValidationMethod.NONE, additionalJsonAnnotations = {JsonTypeName.class}, depluralize = true)
 @JsonSerialize(as = Bundle.class) @JsonDeserialize(builder = Bundle.Builder.class)
 @JsonPropertyOrder({"type", "id", "spec_version", "objects"})
-public interface BundleObject extends GenericValidation, Serializable {
+public interface BundleObject extends GenericValidation, Serializable, StixCustomProperties {
 
     @NotBlank
     @JsonProperty("type")
@@ -67,6 +68,7 @@ default String getSpecVersion() {
     default String toJsonString() {
         JsonNode response = StixParsers.getJsonMapper(true).valueToTree(this);
         ObjectNode responseNode = (ObjectNode) response;
+        //@TODO Refactor as this is causing custom properties to come before the Objects prop:
         responseNode.putArray("objects");
         ArrayNode objects = (ArrayNode) response.get("objects");
 

src/main/java/io/digitalstate/stix/common/StixCustomProperties.java

@@ -5,15 +5,12 @@
 import com.fasterxml.jackson.annotation.JsonUnwrapped;
 import io.digitalstate.stix.validation.contraints.startswith.StartsWith;
 import org.hibernate.validator.constraints.Length;
-import org.immutables.value.Value;
 
 import java.util.Map;
 
 /**
  * Stix Custom Properties
  */
-
-@Value.Style(validationMethod = Value.Style.ValidationMethod.NONE)
 public interface StixCustomProperties {
 
     /**

src/main/java/io/digitalstate/stix/coo/extension/types/NetworkSocketExtensionExt.java

@@ -9,9 +9,9 @@
 import io.digitalstate.stix.validation.contraints.defaulttypevalue.DefaultTypeValue;
 import io.digitalstate.stix.validation.contraints.vocab.Vocab;
 import io.digitalstate.stix.validation.groups.DefaultValuesProcessor;
-import io.digitalstate.stix.vocabularies.NetworkSocketAddressFamilies;
-import io.digitalstate.stix.vocabularies.NetworkSocketProtocolFamilies;
-import io.digitalstate.stix.vocabularies.NetworkSocketTypes;
+import io.digitalstate.stix.vocabulary.vocabularies.NetworkSocketAddressFamilies;
+import io.digitalstate.stix.vocabulary.vocabularies.NetworkSocketProtocolFamilies;
+import io.digitalstate.stix.vocabulary.vocabularies.NetworkSocketTypes;
 import org.immutables.serial.Serial;
 import org.immutables.value.Value;
 

src/main/java/io/digitalstate/stix/coo/extension/types/WindowsPeBinaryFileExtensionExt.java

@@ -13,8 +13,8 @@
 import io.digitalstate.stix.validation.contraints.hashingvocab.HashingVocab;
 import io.digitalstate.stix.validation.contraints.vocab.Vocab;
 import io.digitalstate.stix.validation.groups.DefaultValuesProcessor;
-import io.digitalstate.stix.vocabularies.HashingAlgorithms;
-import io.digitalstate.stix.vocabularies.WindowsPeBinaryTypes;
+import io.digitalstate.stix.vocabulary.vocabularies.HashingAlgorithms;
+import io.digitalstate.stix.vocabulary.vocabularies.WindowsPeBinaryTypes;
 import org.hibernate.validator.constraints.Length;
 import org.immutables.serial.Serial;
 import org.immutables.value.Value;

src/main/java/io/digitalstate/stix/coo/extension/types/WindowsServiceExtensionExt.java

@@ -9,9 +9,9 @@
 import io.digitalstate.stix.validation.contraints.defaulttypevalue.DefaultTypeValue;
 import io.digitalstate.stix.validation.contraints.vocab.Vocab;
 import io.digitalstate.stix.validation.groups.DefaultValuesProcessor;
-import io.digitalstate.stix.vocabularies.WindowsServiceStartTypes;
-import io.digitalstate.stix.vocabularies.WindowsServiceStatuses;
-import io.digitalstate.stix.vocabularies.WindowsServiceTypes;
+import io.digitalstate.stix.vocabulary.vocabularies.WindowsServiceStartTypes;
+import io.digitalstate.stix.vocabulary.vocabularies.WindowsServiceStatuses;
+import io.digitalstate.stix.vocabulary.vocabularies.WindowsServiceTypes;
 import org.immutables.serial.Serial;
 import org.immutables.value.Value;
 

src/main/java/io/digitalstate/stix/coo/json/extension/CyberObservableExtensionsFieldDeserializer.java

@@ -35,7 +35,7 @@ public Set<CyberObservableExtension> deserialize(JsonParser p, DeserializationCo
                 CyberObservableExtension extension = node.traverse(p.getCodec()).readValueAs(CyberObservableExtension.class);
                 extensions.add(extension);
             } catch (IOException e) {
-                e.printStackTrace();
+                throw new IllegalStateException("Cannot deserialize COO extension: ", e);
             }
         });
 

src/main/java/io/digitalstate/stix/coo/json/observables/CyberObservableSetFieldDeserializer.java

@@ -30,7 +30,7 @@ public Set<CyberObservableObject> deserialize(JsonParser p, DeserializationConte
         tree.fieldNames().forEachRemaining(f->{
            ObjectNode node =  (ObjectNode)tree.get(f);
            node.put("observable_object_key", f);
-           System.out.println(node.toString());
+//           System.out.println(node.toString());
             try {
                 CyberObservableObject object = node.traverse(p.getCodec())
                         .readValueAs(CyberObservableObject.class);

src/main/java/io/digitalstate/stix/coo/objects/ArtifactCoo.java

@@ -8,7 +8,7 @@
 import io.digitalstate.stix.validation.contraints.defaulttypevalue.DefaultTypeValue;
 import io.digitalstate.stix.validation.contraints.hashingvocab.HashingVocab;
 import io.digitalstate.stix.validation.groups.DefaultValuesProcessor;
-import io.digitalstate.stix.vocabularies.HashingAlgorithms;
+import io.digitalstate.stix.vocabulary.vocabularies.HashingAlgorithms;
 import org.hibernate.validator.constraints.Length;
 import org.immutables.serial.Serial;
 import org.immutables.value.Value;
@@ -27,7 +27,7 @@
  */
 @Value.Immutable @Serial.Version(1L)
 @DefaultTypeValue(value = "artifact", groups = {DefaultValuesProcessor.class})
-@Value.Style(typeAbstract="*Coo", typeImmutable="*", validationMethod = Value.Style.ValidationMethod.NONE, additionalJsonAnnotations = {JsonTypeName.class}, depluralize = true)
+@Value.Style(typeAbstract="*Coo", typeImmutable="*", validationMethod = Value.Style.ValidationMethod.NONE, additionalJsonAnnotations = {JsonTypeName.class}, depluralize = true, depluralizeDictionary = {"hash:hashes"})
 @JsonTypeName("artifact")
 @JsonSerialize(as = Artifact.class) @JsonDeserialize(builder = Artifact.Builder.class)
 @JsonPropertyOrder({"type", "extensions", "mime_type", "payload_bin", "url", "hashes"})
@@ -47,8 +47,9 @@ public interface ArtifactCoo extends CyberObservableObject {
 
     @JsonProperty("payload_bin")
     @JsonPropertyDescription("Specifies the binary data contained in the artifact as a base64-encoded string.")
-    Optional<@Pattern(regexp = "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)$")
-            String> getPayloadBin();
+    //Removed the @pattern from within the optional until it is clear on the usage and expectation.
+        //@Pattern(regexp = "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)$")
+    Optional<String> getPayloadBin();
 
     /**
      * url-regex

src/main/java/io/digitalstate/stix/coo/objects/DirectoryCoo.java

@@ -42,7 +42,7 @@ public interface DirectoryCoo extends CyberObservableObject {
     @JsonProperty("path_enc")
     @JsonPropertyDescription("Specifies the observed encoding for the path.")
     Optional<@Pattern(regexp = "^[a-zA-Z0-9/\\.+_:-]{2,250}$")
-            String> getName();
+            String> getPathEnc();
 
     @JsonFormat(pattern = StixDataFormats.TIMESTAMP_PATTERN, timezone = "UTC")
     @JsonProperty("created")