Separation of storing and validation operations

This patch separates the build-in type validations from store
operations to allow storing a node even if the value doesn't pass the
validations. To do multiple changes were done:
 - introduces LY_CTX_BASIC_PLUGINS_ONLY flag, which prevents loading of
   any advanced plugins
 - introduces LYPLG_TYPE_STORE_ONLY flag, which skip validation and
   perform just storing
 - introduces LYD_PARSE_STORE_ONLY flag, which implies usage of LYPLG_TYPE_STORE_ONLY
 - introduces options for lyd_new_* API
 - removes lyd_new_(term|list)_(bin|canon) API
 - added sanity checks within lyd_new_(term|list) APIs for proper
   combinations of options
 - refactored lyd_new_* APIs to use common flags to use common options attributes

src/context.c

@@ -283,6 +283,7 @@ ly_ctx_new(const char *search_dir, uint16_t options, struct ly_ctx **new_ctx)
     struct ly_in *in = NULL;
     LY_ERR rc = LY_SUCCESS;
     struct lys_glob_unres unres = {0};
+    ly_bool builtin_plugins_only;
 
     LY_CHECK_ARG_RET(NULL, new_ctx, LY_EINVAL);
 
@@ -293,7 +294,8 @@ ly_ctx_new(const char *search_dir, uint16_t options, struct ly_ctx **new_ctx)
     lydict_init(&ctx->dict);
 
     /* plugins */
-    LY_CHECK_ERR_GOTO(lyplg_init(), LOGINT(NULL); rc = LY_EINT, cleanup);
+    builtin_plugins_only = (options & LY_CTX_BUILTIN_PLUGINS_ONLY) ? 1 : 0;
+    LY_CHECK_ERR_GOTO(lyplg_init(builtin_plugins_only), LOGINT(NULL); rc = LY_EINT, cleanup);
 
     if (options & LY_CTX_LEAFREF_LINKING) {
         ctx->leafref_links_ht = lyht_new(1, sizeof(struct lyd_leafref_links_rec), ly_ctx_ht_leafref_links_equal_cb, NULL, 1);
@@ -1234,8 +1236,7 @@ ly_ctx_get_yanglib_data(const struct ly_ctx *ctx, struct lyd_node **root_p, cons
         LY_CHECK_GOTO(ret = ylib_deviation(cont, mod, 0), error);
 
         /* conformance-type */
-        LY_CHECK_GOTO(ret = lyd_new_term(cont, NULL, "conformance-type", mod->implemented ? "implement" : "import", 0,
-                NULL), error);
+        LY_CHECK_GOTO(ret = lyd_new_term(cont, NULL, "conformance-type", mod->implemented ? "implement" : "import", 0, NULL), error);
 
         /* submodule list */
         LY_CHECK_GOTO(ret = ylib_submodules(cont, mod->parsed, 0), error);

src/context.h

@@ -202,6 +202,13 @@ struct ly_ctx;
                                         'require-instance false;'. It also enables usage of
                                         [lyd_leafref_get_links](@ref lyd_leafref_get_links) and
                                         [lyd_leafref_link_node_tree](@ref lyd_leafref_link_node_tree) APIs. */
+#define LY_CTX_BUILTIN_PLUGINS_ONLY 0x0800 /**< By default, context loads all available built-in plugins and extensions. This
+                                        options prohibits loading of built-in non-YANG plugin types and extensions (ipv4-address,
+                                        ipv6-address, etc.). Instead the value is processed by built-in YANG plugins (usually string)
+                                        with all its behavioral implications as hex-string comparison becoming case-sensitive.
+                                        Change of this flag during the lifetime of context is not fully supported. Once the non-YANG
+                                        plugins or extensions are loaded during context creation, they will be used regardless of this
+                                        flag. Therefore it is recommended to set this flag during the first context creation. */
 
 /** @} contextoptions */
 

src/diff.c

@@ -154,7 +154,7 @@ lyd_diff_add_create_nested_userord(struct lyd_node *node)
     }
 
     /* create the metadata */
-    LY_CHECK_GOTO(rc = lyd_new_meta(NULL, node, NULL, meta_name, meta_val, 0, NULL), cleanup);
+    LY_CHECK_GOTO(rc = lyd_new_meta(NULL, node, NULL, meta_name, meta_val, LYD_NEW_VAL_STORE_ONLY, NULL), cleanup);
 
 cleanup:
     free(dyn);
@@ -382,7 +382,7 @@ lyd_diff_add(const struct lyd_node *node, enum lyd_diff_op op, const char *orig_
             }
 
             /* set the none operation */
-            LY_CHECK_RET(lyd_new_meta(NULL, elem, NULL, "yang:operation", "none", 0, NULL));
+            LY_CHECK_RET(lyd_new_meta(NULL, elem, NULL, "yang:operation", "none", LYD_NEW_VAL_STORE_ONLY, NULL));
         }
 
         dup = diff_parent;
@@ -424,12 +424,12 @@ lyd_diff_add(const struct lyd_node *node, enum lyd_diff_op op, const char *orig_
 
         /* add parent operation, if any */
         if (diff_parent && (diff_parent != dup)) {
-            LY_CHECK_RET(lyd_new_meta(NULL, diff_parent, NULL, "yang:operation", "none", 0, NULL));
+            LY_CHECK_RET(lyd_new_meta(NULL, diff_parent, NULL, "yang:operation", "none", LYD_NEW_VAL_STORE_ONLY, NULL));
         }
     }
 
     /* add subtree operation */
-    LY_CHECK_RET(lyd_new_meta(NULL, dup, NULL, "yang:operation", lyd_diff_op2str(op), 0, NULL));
+    LY_CHECK_RET(lyd_new_meta(NULL, dup, NULL, "yang:operation", lyd_diff_op2str(op), LYD_NEW_VAL_STORE_ONLY, NULL));
 
     if (op == LYD_DIFF_OP_CREATE) {
         /* all nested user-ordered (leaf-)lists need special metadata for create op */
@@ -443,37 +443,37 @@ lyd_diff_add(const struct lyd_node *node, enum lyd_diff_op op, const char *orig_
 
     /* orig-default */
     if (orig_default) {
-        LY_CHECK_RET(lyd_new_meta(NULL, dup, NULL, "yang:orig-default", orig_default, 0, NULL));
+        LY_CHECK_RET(lyd_new_meta(NULL, dup, NULL, "yang:orig-default", orig_default, LYD_NEW_VAL_STORE_ONLY, NULL));
     }
 
     /* orig-value */
     if (orig_value) {
-        LY_CHECK_RET(lyd_new_meta(NULL, dup, NULL, "yang:orig-value", orig_value, 0, NULL));
+        LY_CHECK_RET(lyd_new_meta(NULL, dup, NULL, "yang:orig-value", orig_value, LYD_NEW_VAL_STORE_ONLY, NULL));
     }
 
     /* key */
     if (key) {
-        LY_CHECK_RET(lyd_new_meta(NULL, dup, NULL, "yang:key", key, 0, NULL));
+        LY_CHECK_RET(lyd_new_meta(NULL, dup, NULL, "yang:key", key, LYD_NEW_VAL_STORE_ONLY, NULL));
     }
 
     /* value */
     if (value) {
-        LY_CHECK_RET(lyd_new_meta(NULL, dup, NULL, "yang:value", value, 0, NULL));
+        LY_CHECK_RET(lyd_new_meta(NULL, dup, NULL, "yang:value", value, LYD_NEW_VAL_STORE_ONLY, NULL));
     }
 
     /* position */
     if (position) {
-        LY_CHECK_RET(lyd_new_meta(NULL, dup, NULL, "yang:position", position, 0, NULL));
+        LY_CHECK_RET(lyd_new_meta(NULL, dup, NULL, "yang:position", position, LYD_NEW_VAL_STORE_ONLY, NULL));
     }
 
     /* orig-key */
     if (orig_key) {
-        LY_CHECK_RET(lyd_new_meta(NULL, dup, NULL, "yang:orig-key", orig_key, 0, NULL));
+        LY_CHECK_RET(lyd_new_meta(NULL, dup, NULL, "yang:orig-key", orig_key, LYD_NEW_VAL_STORE_ONLY, NULL));
     }
 
     /* orig-position */
     if (orig_position) {
-        LY_CHECK_RET(lyd_new_meta(NULL, dup, NULL, "yang:orig-position", orig_position, 0, NULL));
+        LY_CHECK_RET(lyd_new_meta(NULL, dup, NULL, "yang:orig-position", orig_position, LYD_NEW_VAL_STORE_ONLY, NULL));
     }
 
     return LY_SUCCESS;
@@ -1460,7 +1460,7 @@ lyd_diff_change_op(struct lyd_node *node, enum lyd_diff_op op)
     lyd_diff_del_meta(node, "operation");
 
     if (node->schema) {
-        return lyd_new_meta(LYD_CTX(node), node, NULL, "yang:operation", lyd_diff_op2str(op), 0, NULL);
+        return lyd_new_meta(LYD_CTX(node), node, NULL, "yang:operation", lyd_diff_op2str(op), LYD_NEW_VAL_STORE_ONLY, NULL);
     } else {
         return lyd_new_attr(node, "yang", "operation", lyd_diff_op2str(op), NULL);
     }
@@ -1685,7 +1685,7 @@ lyd_diff_merge_create(struct lyd_node **diff_match, struct lyd_node **diff, enum
 
                 /* current value is the previous one (meta) */
                 LY_CHECK_RET(lyd_new_meta(LYD_CTX(src_diff), *diff_match, NULL, "yang:orig-value",
-                        lyd_get_value(*diff_match), 0, NULL));
+                        lyd_get_value(*diff_match), LYD_NEW_VAL_STORE_ONLY, NULL));
 
                 /* update the value itself */
                 LY_CHECK_RET(lyd_change_term(*diff_match, lyd_get_value(src_diff)));
@@ -1699,10 +1699,10 @@ lyd_diff_merge_create(struct lyd_node **diff_match, struct lyd_node **diff, enum
                 to_free = *diff_match;
                 *diff_match = src_dup;
 
-                r = lyd_new_meta(ctx, src_dup, NULL, "yang:orig-value", lyd_get_value(to_free), 0, NULL);
+                r = lyd_new_meta(ctx, src_dup, NULL, "yang:orig-value", lyd_get_value(to_free), LYD_NEW_VAL_STORE_ONLY, NULL);
                 lyd_free_tree(to_free);
                 LY_CHECK_RET(r);
-                LY_CHECK_RET(lyd_new_meta(ctx, src_dup, NULL, "yang:operation", lyd_diff_op2str(LYD_DIFF_OP_REPLACE), 0, NULL));
+                LY_CHECK_RET(lyd_new_meta(ctx, src_dup, NULL, "yang:operation", lyd_diff_op2str(LYD_DIFF_OP_REPLACE), LYD_NEW_VAL_STORE_ONLY, NULL));
             }
         } else {
             /* deleted + created -> operation NONE */
@@ -1713,7 +1713,7 @@ lyd_diff_merge_create(struct lyd_node **diff_match, struct lyd_node **diff, enum
         if ((*diff_match)->schema->nodetype & LYD_NODE_TERM) {
             /* add orig-dflt metadata */
             LY_CHECK_RET(lyd_new_meta(LYD_CTX(src_diff), *diff_match, NULL, "yang:orig-default",
-                    trg_flags & LYD_DEFAULT ? "true" : "false", 0, NULL));
+                    trg_flags & LYD_DEFAULT ? "true" : "false", LYD_NEW_VAL_STORE_ONLY, NULL));
 
             /* update dflt flag itself */
             (*diff_match)->flags &= ~LYD_DEFAULT;
@@ -1763,7 +1763,7 @@ lyd_diff_merge_delete(struct lyd_node *diff_match, enum lyd_diff_op cur_op, cons
         if (diff_match->schema->nodetype & LYD_NODE_TERM) {
             /* add orig-default meta because it is expected */
             LY_CHECK_RET(lyd_new_meta(LYD_CTX(src_diff), diff_match, NULL, "yang:orig-default",
-                    src_diff->flags & LYD_DEFAULT ? "true" : "false", 0, NULL));
+                    src_diff->flags & LYD_DEFAULT ? "true" : "false", LYD_NEW_VAL_STORE_ONLY, NULL));
         }
         break;
     case LYD_DIFF_OP_REPLACE:

src/parser_common.c

@@ -284,8 +284,10 @@ lyd_parser_create_term(struct lyd_ctx *lydctx, const struct lysc_node *schema, c
 {
     LY_ERR r;
     ly_bool incomplete;
+    ly_bool store_only = (lydctx->parse_opts & LYD_PARSE_STORE_ONLY) ? 1 : 0;
 
-    if ((r = lyd_create_term(schema, value, value_len, 1, dynamic, format, prefix_data, hints, &incomplete, node))) {
+    if ((r = lyd_create_term(schema, value, value_len, 1, store_only, dynamic, format, prefix_data,
+            hints, &incomplete, node))) {
         if (lydctx->data_ctx->ctx != schema->module->ctx) {
             /* move errors to the main context */
             ly_err_move(schema->module->ctx, (struct ly_ctx *)lydctx->data_ctx->ctx);
@@ -308,6 +310,7 @@ lyd_parser_create_meta(struct lyd_ctx *lydctx, struct lyd_node *parent, struct l
     char *dpath = NULL, *path = NULL;
     ly_bool incomplete;
     struct lyd_meta *first = NULL;
+    ly_bool store_only = (lydctx->parse_opts & LYD_PARSE_STORE_ONLY) ? 1 : 0;
 
     if (meta && *meta) {
         /* remember the first metadata */
@@ -323,7 +326,7 @@ lyd_parser_create_meta(struct lyd_ctx *lydctx, struct lyd_node *parent, struct l
     }
     ly_log_location(NULL, NULL, path, NULL);
 
-    LY_CHECK_GOTO(rc = lyd_create_meta(parent, meta, mod, name, name_len, value, value_len, 1, dynamic, format,
+    LY_CHECK_GOTO(rc = lyd_create_meta(parent, meta, mod, name, name_len, value, value_len, 1, store_only, dynamic, format,
             prefix_data, hints, ctx_node, 0, &incomplete), cleanup);
 
     if (incomplete && !(lydctx->parse_opts & LYD_PARSE_ONLY)) {

src/parser_data.h

@@ -134,6 +134,7 @@ struct ly_in;
  * - when statements on existing nodes are evaluated, if not satisfied, a validation error is raised,
  * - invalid multiple data instances/data from several cases cause a validation error,
  * - implicit nodes (NP containers and default values) are added.
+ * - Validations based on leaf/leaf-list types restriction is being done regardless of setting LYD_PARSE_ONLY
  * @{
  */
 /* note: keep the lower 16bits free for use by LYD_VALIDATE_ flags. They are not supposed to be combined together,
@@ -171,6 +172,8 @@ struct ly_in;
 #define LYD_PARSE_NO_NEW 0x1000000          /**< Do not set ::LYD_NEW (non-validated node flag) for any nodes. Use
                                                  when parsing validated data to skip some validation tasks and modify
                                                  some validation behavior (auto-deletion of cases). */
+#define LYD_PARSE_STORE_ONLY 0x2000000      /**< Perform only storing operation, no validation based on leaf/leaf-list type
+                                                 restrictions will be performed. */
 
 #define LYD_PARSE_OPTS_MASK 0xFFFF0000      /**< Mask for all the LYD_PARSE_ options. */
 

src/path.c

@@ -710,8 +710,8 @@ ly_path_compile_predicate(const struct ly_ctx *ctx, const struct lysc_node *cur_
                 if (key) {
                     LOG_LOCSET(key, NULL);
                 }
-                ret = lyd_value_store(ctx, &p->value, ((struct lysc_node_leaf *)key)->type, val, val_len, 0, NULL,
-                        format, prefix_data, LYD_HINT_DATA, key, NULL);
+                ret = lyd_value_store(ctx, &p->value, ((struct lysc_node_leaf *)key)->type, val, val_len, 0, 0,
+                        NULL, format, prefix_data, LYD_HINT_DATA, key, NULL);
                 LOG_LOCBACK(key ? 1 : 0, 0);
                 LY_CHECK_ERR_GOTO(ret, p->value.realtype = NULL, cleanup);
 
@@ -776,8 +776,8 @@ ly_path_compile_predicate(const struct ly_ctx *ctx, const struct lysc_node *cur_
         if (ctx_node) {
             LOG_LOCSET(ctx_node, NULL);
         }
-        ret = lyd_value_store(ctx, &p->value, ((struct lysc_node_leaflist *)ctx_node)->type, val, val_len, 0, NULL,
-                format, prefix_data, LYD_HINT_DATA, ctx_node, NULL);
+        ret = lyd_value_store(ctx, &p->value, ((struct lysc_node_leaflist *)ctx_node)->type, val, val_len, 0, 0,
+                NULL, format, prefix_data, LYD_HINT_DATA, ctx_node, NULL);
         LOG_LOCBACK(ctx_node ? 1 : 0, 0);
         LY_CHECK_ERR_GOTO(ret, p->value.realtype = NULL, cleanup);
         ++(*tok_idx);
@@ -1356,7 +1356,7 @@ ly_path_eval_partial(const struct ly_path *path, const struct lyd_node *start, c
             case LY_PATH_PREDTYPE_LIST_VAR:
             case LY_PATH_PREDTYPE_LIST:
                 /* we will use hashes to find one list instance */
-                LY_CHECK_RET(lyd_create_list(path[u].node, path[u].predicates, vars, &target));
+                LY_CHECK_RET(lyd_create_list(path[u].node, path[u].predicates, vars, 1, &target));
                 lyd_find_sibling_first(start, target, &node);
                 lyd_free_tree(target);
                 break;

src/plugins.c

@@ -442,7 +442,7 @@ plugins_insert_dir(enum LYPLG type)
 #endif
 
 LY_ERR
-lyplg_init(void)
+lyplg_init(ly_bool builtin_plugins_only)
 {
     LY_ERR ret;
 
@@ -468,34 +468,36 @@ lyplg_init(void)
     LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_string), error);
     LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_union), error);
 
-    /* yang */
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_instanceid_keys), error);
-
-    /* ietf-inet-types */
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_ipv4_address), error);
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_ipv4_address_no_zone), error);
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_ipv6_address), error);
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_ipv6_address_no_zone), error);
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_ipv4_prefix), error);
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_ipv6_prefix), error);
-
-    /* ietf-yang-types */
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_date_and_time), error);
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_hex_string), error);
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_xpath10), error);
-
-    /* ietf-netconf-acm */
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_node_instanceid), error);
-
-    /* lyds_tree */
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_lyds_tree), error);
-
-    /* internal extensions */
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_EXTENSION, plugins_metadata), error);
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_EXTENSION, plugins_nacm), error);
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_EXTENSION, plugins_yangdata), error);
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_EXTENSION, plugins_schema_mount), error);
-    LY_CHECK_GOTO(ret = plugins_insert(LYPLG_EXTENSION, plugins_structure), error);
+    if (!builtin_plugins_only) {
+        /* yang */
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_instanceid_keys), error);
+
+        /* ietf-inet-types */
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_ipv4_address), error);
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_ipv4_address_no_zone), error);
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_ipv6_address), error);
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_ipv6_address_no_zone), error);
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_ipv4_prefix), error);
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_ipv6_prefix), error);
+
+        /* ietf-yang-types */
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_date_and_time), error);
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_hex_string), error);
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_xpath10), error);
+
+        /* ietf-netconf-acm */
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_node_instanceid), error);
+
+        /* lyds_tree */
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_TYPE, plugins_lyds_tree), error);
+
+        /* internal extensions */
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_EXTENSION, plugins_metadata), error);
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_EXTENSION, plugins_nacm), error);
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_EXTENSION, plugins_yangdata), error);
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_EXTENSION, plugins_schema_mount), error);
+        LY_CHECK_GOTO(ret = plugins_insert(LYPLG_EXTENSION, plugins_structure), error);
+    }
 
 #ifndef STATIC
     /* external types */

src/plugins_internal.h

@@ -48,11 +48,12 @@
  *
  * Covers both the types and extensions plugins.
  *
+ * @param[in] builtin_plugins_only Whether to load only built-in YANG plugin types and extensions.
  * @return LY_SUCCESS in case of success
  * @return LY_EINT in case of internal error
  * @return LY_EMEM in case of memory allocation failure.
  */
-LY_ERR lyplg_init(void);
+LY_ERR lyplg_init(ly_bool builtin_plugins_only);
 
 /**
  * @brief Remove (unload) all the plugins currently available.

src/plugins_types.h

@@ -475,6 +475,7 @@ LIBYANG_API_DECL LY_ERR lyplg_type_print_xpath10_value(const struct lyd_value_xp
 #define LYPLG_TYPE_STORE_IMPLEMENT 0x02 /**< If a foreign module is needed to be implemented to successfully instantiate
                                              the value, make the module implemented. */
 #define LYPLG_TYPE_STORE_IS_UTF8   0x04 /**< The value is guaranteed to be a valid UTF-8 string, if applicable for the type. */
+#define LYPLG_TYPE_STORE_ONLY      0x08 /**< The value is stored only. The validation must be done using [validate](@ref lyplg_type_validate_clb) */
 /** @} plugintypestoreopts */
 
 /**