lib/src/clixon_yang_type.c

/*
 *
  ***** BEGIN LICENSE BLOCK *****

  Copyright (C) 2009-2016 Olof Hagsand and Benny Holmgren
  Copyright (C) 2017-2019 Olof Hagsand
  Copyright (C) 2020 Olof Hagsand and Rubicon Communications, LLC(Netgate)

  This file is part of CLIXON.

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.

  Alternatively, the contents of this file may be used under the terms of
  the GNU General Public License Version 3 or later (the "GPL"),
  in which case the provisions of the GPL are applicable instead
  of those above. If you wish to allow use of your version of this file only
  under the terms of the GPL, and not to allow others to
  use your version of this file under the terms of Apache License version 2,
  indicate your decision by deleting the provisions above and replace them with
  the  notice and other provisions required by the GPL. If you do not delete
  the provisions above, a recipient may use your version of this file under
  the terms of any one of the Apache License version 2 or the GPL.

  ***** END LICENSE BLOCK *****

 * Yang type related functions
 * Part of this is type resolving which is pretty complex
 *                   +--> yang_type_cache_set
 * (called at parse) |
 * ys_resolve_type  --+     ys_populate_range, yang_enum_int_value(NULL)
 *                     \    |  cml
 *                      v   v  v
 * yang_type_get       -->  yang_type_resolve --> resolve_restrictions
 * (leaf(list) front)       (recursive core fn)   (regexps, length, ranges, ...)
 * ^  ^                     ^  ^
 * |  |                     |  |
 * |  yang2cli_var          |  yang2cli_var_union_one
 * ys_cv_validate---+      ys_cv_validate_union_one
 * |                 \    /
 * |                  \  /    yang_type_cache_regex_set
 * ys_populate_leaf,   +--> compile_pattern2regexp (compile regexps)
 * xml_cv_cache (NULL) +--> cv_validate1 --> cv_validate_pattern (exec regexps)
 * yang_type2cv (simplified)
 *
 * NOTE
 * 1) ys_cv_validate/ys_cv_validate_union_one and
 *    yang2cli_var/yang2cli_var_union_one can unify?
 * 2) Cache of regex is set in ys_cv_validate - not in ys_reolve_parse
 *    This is because trees are copied in yang_parse_post after ys_reolve_type
 *    is called, and regexps (void*) cannot be copied (COMPLEX)
 * 3) We know I think when cache is set and when it is not set in the calls
 *    to yang_type_resolve. maybe we should make code easier by a separate
 *    yang_type_resolve_cache() call?
 */

#ifdef HAVE_CONFIG_H
#include "clixon_config.h" /* generated by config & autoconf */
#endif

#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <inttypes.h>
#include <limits.h>
#include <ctype.h>
#include <string.h>
#include <arpa/inet.h>
#include <regex.h>
#include <syslog.h>
#include <assert.h>
#include <regex.h>
#include <netinet/in.h>
#include <sys/param.h>

/* cligen */
#include <cligen/cligen.h>

/* clicon */
#include "clixon_log.h"
#include "clixon_err.h"
#include "clixon_string.h"
#include "clixon_queue.h"
#include "clixon_hash.h"
#include "clixon_handle.h"
#include "clixon_regex.h"
#include "clixon_hash.h"
#include "clixon_yang.h"
#include "clixon_xml.h"
#include "clixon_xml_nsctx.h"
#include "clixon_yang_module.h"
#include "clixon_plugin.h"
#include "clixon_options.h"
#include "clixon_yang_type.h"

/*
 * Local types and variables
 */

/* Mapping between yang types <--> cligen types
   Note, first match used wne translating from cv to yang --> order is significant */
static const map_str2int ytmap[] = {
    {"int32",       CGV_INT32},  /* NOTE, first match on right is significant, dont move */
    {"string",      CGV_STRING}, /* NOTE, first match on right is significant, dont move */
    {"string",      CGV_REST},   /* For cv -> yang translation of rest */
    {"binary",      CGV_STRING},
    {"bits",        CGV_STRING},
    {"boolean",     CGV_BOOL},
    {"decimal64",   CGV_DEC64},
    {"empty",       CGV_VOID},  /* May not include any content */
    {"enumeration", CGV_STRING},
    {"identityref", CGV_STRING},  /* XXX */
    {"instance-identifier", CGV_STRING}, /* XXX */
    {"int8",        CGV_INT8},
    {"int16",       CGV_INT16},
    {"int64",       CGV_INT64},
    {"leafref",     CGV_STRING},  /* XXX */
    {"uint8",       CGV_UINT8},
    {"uint16",      CGV_UINT16},
    {"uint32",      CGV_UINT32},
    {"uint64",      CGV_UINT64},
    {"union",       CGV_REST},  /* Is replaced by actual type */
    {NULL,         -1}
};

/*! Mapping from yang string types --> cligen types
 * @note not 100% same as map_str2int since it has significant order AND
 *       string->CGV_REST entry removed
 */
static const map_str2int ytmap2[] = {
    {"binary",      CGV_STRING},
    {"bits",        CGV_STRING},
    {"boolean",     CGV_BOOL},
    {"decimal64",   CGV_DEC64},
    {"empty",       CGV_VOID},  /* May not include any content */
    {"enumeration", CGV_STRING},
    {"identityref", CGV_STRING},  /* XXX */
    {"instance-identifier", CGV_STRING}, /* XXX */
    {"int16",       CGV_INT16},
    {"int32",       CGV_INT32},
    {"int64",       CGV_INT64},
    {"int8",        CGV_INT8},
    {"leafref",     CGV_STRING},  /* XXX */
    {"string",      CGV_STRING},
    {"uint16",      CGV_UINT16},
    {"uint32",      CGV_UINT32},
    {"uint64",      CGV_UINT64},
    {"uint8",       CGV_UINT8},
    {"union",       CGV_REST},  /* Is replaced by actual type */
    {NULL,         -1}
};

/* return 1 if built-in, 0 if not */
static int
yang_builtin(char *type)
{
    if (clicon_str2int_search(ytmap2, type, (sizeof(ytmap)/sizeof(map_str2int))-2) != -1)
	return 1;
    return 0;
}

/*! Compile yang patterns in string form to regex compiled void* form
 * and re-store into "patterns" cvec.
 * This is done here instead of deep in resolve code (resolve_restrictions)
 * since it id dependent on clicon_handle.
 * The downside is that all accesses to "patterns" must pass via the cache.
 * If calls to yang_type_resolve is made without the cache is set, will be
 * wrong.
 * @see match_regexp  in cligen code
 * @see yang_type_resolve_restrictions  where patterns is set
 */
static int
compile_pattern2regexp(clicon_handle h,
		       cvec         *patterns,
		       cvec         *regexps)
{
    int     retval = -1;
    cg_var *pcv; /* pattern cv */
    cg_var *rcv; /* regexp cv */
    void   *re = NULL;
    int     ret;
    char   *pattern;

    pcv = NULL;
    while ((pcv = cvec_each(patterns, pcv)) != NULL){
	pattern = cv_string_get(pcv);
	/* Compile yang pattern. handle necessary to select regex engine */
	if ((ret = regex_compile(h, pattern, &re)) < 0)
	    goto done;
	if (ret == 0){
	    clicon_err(OE_YANG, errno, "regexp compile fail: \"%s\"",
		       pattern);
	    goto done;
	    break;
	}
	if ((rcv = cvec_add(regexps, CGV_VOID)) == NULL){
	    clicon_err(OE_UNIX, errno, "cvec_add");
	    goto done;
	}
	if (re != NULL)
	    cv_void_set(rcv, re);
	re = NULL;
	/* invert pattern check */
	if (cv_flag(pcv, V_INVERT))
	    cv_flag_set(rcv, V_INVERT);
    }
    retval = 1;
 done:
    return retval;
}

/*! Resolve types: populate type caches
 * @param[in]  ys  This is a type statement
 * @param[in]  arg Not used
 * Typically only called once when loading te yang type system.
 * @note unions not cached
 */
int
ys_resolve_type(yang_stmt    *ys,
		void         *arg)
{
    //    clicon_handle     h = (clicon_handle)arg;
    int               retval = -1;
    int               options = 0x0;
    cvec             *cvv = NULL;
    cvec             *patterns = NULL;
    uint8_t           fraction = 0;
    yang_stmt        *resolved = NULL;

    if (yang_keyword_get(ys) != Y_TYPE){
	clicon_err(OE_YANG, EINVAL, "Expected Y_TYPE");
	goto done;
    }
    if ((patterns = cvec_new(0)) == NULL){
       clicon_err(OE_UNIX, errno, "cvec_new");
       goto done;
    }
    /* Recursively resolve ys -> resolve with restrictions(options, etc)
     * Note that the resolved type could be ys itself.
     */
    if (yang_type_resolve(yang_parent_get(ys), yang_parent_get(ys),
			  ys, &resolved,
			  &options, &cvv, patterns, NULL, &fraction) < 0)
	goto done;

    /* Cache the type resolving locally. Only place where this is done.
     * Why not do it in yang_type_resolve? (compile regexps needs clicon_handle)
     */
    if (yang_type_cache_set(ys, resolved, options, cvv,
			    patterns, fraction) < 0)
	goto done;
    retval = 0;
 done:
    if (patterns)
	cvec_free(patterns);
    return retval;
}

/*! Translate from a yang type to a cligen variable type
 *
 * Currently many built-in types from RFC6020 and some RFC6991 types.
 * But not all, neither built-in nor 6991.
 * Also, there is no support for derived types, eg yang typedefs.
 * See 4.2.4 in RFC6020
 * Return 0 if no match but set cv_type to CGV_ERR
 */
int
yang2cv_type(char         *ytype,
	     enum cv_type *cv_type)
{
    int                ret;

    *cv_type = CGV_ERR;
    /* built-in types */
    if ((ret = clicon_str2int_search(ytmap2, ytype, (sizeof(ytmap)/sizeof(map_str2int))-2)) != -1){
	*cv_type = ret;
	return 0;
    }
    return 0;
}

/*! Translate from a cligen variable type to a yang type
 */
char *
cv2yang_type(enum cv_type cv_type)
{
    char                *ytype;
    const char          *str;

    ytype = "empty";
    /* built-in types */
    if ((str = clicon_int2str(ytmap, cv_type)) != NULL)
	return (char*)str;

    /* special derived types */
    if (cv_type == CGV_IPV4ADDR) /* RFC6991 */
	return "ipv4-address";

    if (cv_type == CGV_IPV6ADDR) /* RFC6991 */
	return "ipv6-address";

    if (cv_type == CGV_IPV4PFX) /* RFC6991 */
	return "ipv4-prefix";

    if (cv_type == CGV_IPV6PFX) /* RFC6991 */
	return "ipv6-prefix";

    if (cv_type == CGV_TIME) /* RFC6991 */
	return "date-and-time";

    if (cv_type == CGV_MACADDR) /* RFC6991 */
	return "mac-address";

    if (cv_type == CGV_UUID) /* RFC6991 */
	return "uuid";

    return ytype;
}

/*! Translate from yang type -> cligen type, after yang resolve has been made.
 * handle case where yang resolve did not succedd (rtype=NULL) and then try
 * to find special cligen types such as ipv4addr.
 * not true yang types
 * @param[in]  origtype Name of original type
 * @param[in]  restype  Resolved type. May be null, in that case origtype is used
 * @param[in]  ys       Yang stmt of original resolving node
 * @param[out] cvtype   Translation from resolved type
 * @note Thereis a kludge for handling direct translations of native cligen types
 */
int
clicon_type2cv(char         *origtype,
	       char         *restype,
	       yang_stmt    *ys,
	       enum cv_type *cvtype)
{
    int retval = -1;
    yang_stmt *ym;

    *cvtype = CGV_ERR;
    ym = ys_module(ys);
    if (restype != NULL){
	yang2cv_type(restype, cvtype);
	if (*cvtype == CGV_ERR){
	    clicon_err(OE_YANG, 0, "%s: \"%s\" type not translated",
		       yang_argument_get(ym), restype);
	    goto done;
	}
    }
    else{
	/*
	 * Not resolved, but we can use special cligen types, eg ipv4addr
	 * Note this is a kludge or at least if we intend of using rfc types
	 */
	yang2cv_type(origtype, cvtype);
	if (*cvtype == CGV_ERR){
	    clicon_err(OE_YANG, 0, "%s:\"%s\": type not resolved",
		       yang_argument_get(ym), origtype);
	    goto done;
	}
    }
    retval = 0;
  done:
    return retval;
}

/*! Validate CLIgen variable with pattern statements
 * @param[in]  h       Clicon handle
 * @param[in]  regexps Vector of compiled regexps
 * @param[out] reason  If given, and return value is 0, contains malloced string
 * @retval -1  Error (fatal), with errno set to indicate error
 * @retval 0   Validation not OK, malloced reason is returned. Free reason with free()
 * @retval 1   Validation OK
 */
static int
cv_validate_pattern(clicon_handle h,
		    cvec         *regexps,
		    yang_stmt    *yrestype,
		    char         *str,
		    char        **reason)
{
    int     retval = -1;
    cg_var *cvr;
    void   *re = NULL;
    int     ret;

    cvr = NULL; /* Loop over compiled regexps */
    while ((cvr = cvec_each(regexps, cvr)) != NULL){
	re = cv_void_get(cvr);
	if ((ret = regex_exec(h, re, str?str:"")) < 0)
	    goto done;
	if (cv_flag(cvr, V_INVERT))
	    ret = !ret; /* swap 0 and 1 */
	if (ret == 0){
	    if (reason)
		*reason = cligen_reason("regexp match fail: pattern does not match %s",
					str);
	    goto fail;
	    break;
	}
    }
    retval = 1; /* match */
 done:
    return retval;
 fail:
    retval = 0; /* validation failed */
    goto done;

}

/* cf cligen/cligen_var.c */
#define range_check(i, rmin, rmax, type)       \
    ((rmin && (i) < cv_##type##_get(rmin)) ||  \
     (rmax && (i) > cv_##type##_get(rmax)))


/*! Error messsage for int violating ranges
 * @note contains kludge - duplicate loop
 */
static int
outofrange(cg_var *cv0,
	   cvec   *cvv,
	   char  **reason)
{
    int     retval = -1;
    cbuf   *cb = NULL;
    cg_var *cv1;
    cg_var *cv2;
    int     i;

    if ((cb = cbuf_new()) == NULL)
	goto done;
    cprintf(cb, "Number ");
    cv2cbuf(cv0, cb);
    cprintf(cb, " out of range: ");
    /* Kludge: need to repeat the same loop as in the main function in
       cv_validate1 */
    i = 0;
    while (i<cvec_len(cvv)){
	cv1 = cvec_i(cvv, i++); /* Increment to check for max pair */
	if (strcmp(cv_name_get(cv1),"range_min") != 0){
	    clicon_err(OE_YANG, EINVAL, "Internal error, expected range_min");
	    goto done;
	}
	if (i<cvec_len(cvv) &&
	    (cv2 = cvec_i(cvv, i)) != NULL &&
	    strcmp(cv_name_get(cv2),"range_max") == 0){
	    i++;
	}
	else
	    cv2 = cv1;
	if (i>2)
	    cprintf(cb, ", ");
	cv2cbuf(cv1, cb);
	cprintf(cb, " - ");
	cv2cbuf(cv2, cb);
    }
    if (reason && (*reason = strdup(cbuf_get(cb))) == NULL)
	goto done;
    if (cb)
	cbuf_free(cb);
    retval = 0;
 done:
    return retval;
}

/*! Error messsage for string violating string limits
 * @note contains kludge - duplicate loop
 */
static int
outoflength(uint64_t    u64,
	    cvec       *cvv,
	    char      **reason)
{
    int     retval = -1;
    cbuf   *cb = NULL;
    cg_var *cv1;
    cg_var *cv2;
    int     i;

    if ((cb = cbuf_new()) == NULL)
	goto done;
    cprintf(cb, "String length %" PRIu64 " out of range: ", u64);

    /* Kludge: need to repeat the same loop as in the main function in
       cv_validate1 */
    i = 0;
    while (i<cvec_len(cvv)){
	cv1 = cvec_i(cvv, i++); /* Increment to check for max pair */
	if (strcmp(cv_name_get(cv1),"range_min") != 0){
	    clicon_err(OE_YANG, EINVAL, "Internal error, expected range_min");
	    goto done;
	}
	if (i<cvec_len(cvv) &&
	    (cv2 = cvec_i(cvv, i)) != NULL &&
	    strcmp(cv_name_get(cv2),"range_max") == 0){
	    i++;
	}
	else
	    cv2 = cv1;
	if (i>2)
	    cprintf(cb, ", ");
	cv2cbuf(cv1, cb);
	cprintf(cb, " - ");
	cv2cbuf(cv2, cb);
    }
    if (reason && (*reason = strdup(cbuf_get(cb))) == NULL)
	goto done;
    if (cb)
	cbuf_free(cb);
    retval = 0;
 done:
    return retval;
}

/*! Validate CLIgen variable
 * @param[in]  h       Clicon handle
 * @param[in]  cv      A cligen variable to validate. This is a correctly parsed cv.
 * @param[in]  cvtype  Resolved type of cv
 *                     string describing reason why validation failed.
 * @param[in]  regexps Vector of compiled regexps
 * @param[out] reason  If given, and return value is 0, contains malloced str

 * @retval    -1       Error (fatal), with errno set to indicate error
 * @retval     0       Validation not OK, malloced reason is returned. Free reason with free()
 * @retval     1       Validation OK
 * @note reason if given must be freed by caller
 * @see cv_validate Corresponding type check in cligen
 */
static int
cv_validate1(clicon_handle h,
	     cg_var      *cv,
	     enum cv_type cvtype,
	     int          options,
	     cvec        *cvv,
	     cvec        *regexps,
	     yang_stmt   *yrestype,
	     char        *restype,
	     char       **reason)
{
    int             retval = 1; /* OK */
    cg_var         *cv1;
    cg_var         *cv2;
    int             ret;
    yang_stmt      *yi = NULL;
    char           *str = NULL;
    int             found;
    char          **vec = NULL;
    int             nvec;
    char           *v;
    uint64_t        uu = 0;
    int64_t         ii = 0;
    int             i;
    int             reti; /* must keep signed, unsigned and string retval */
    int             retu; /* separated due to different error handling */
    int             rets;

    if (reason && *reason){
	free(*reason);
	*reason = NULL;
    }
    /* check options first for length and range */
    if ((options & YANG_OPTIONS_RANGE) != 0 ||
	(options & YANG_OPTIONS_LENGTH) != 0){
	i = 0;
	while (i<cvec_len(cvv)){
	    cv1 = cvec_i(cvv, i++); /* Increment to check for max pair */
	    if (strcmp(cv_name_get(cv1),"range_min") != 0){
		clicon_err(OE_YANG, EINVAL, "Internal error, expected range_min");
		goto done;
	    }
	    if (i<cvec_len(cvv) &&
		(cv2 = cvec_i(cvv, i)) != NULL &&
		strcmp(cv_name_get(cv2),"range_max") == 0){
		i++;
	    }
	    else
		cv2 = cv1;
	    reti = 0; retu = 0; rets = 0;
	    switch (cvtype){
	    case CGV_INT8:
		ii =  cv_int8_get(cv);
		reti = range_check(ii, cv1, cv2, int8);
		break;
	    case CGV_INT16:
		ii =  cv_int16_get(cv);
		reti = range_check(ii, cv1, cv2, int16);
		break;
	    case CGV_INT32:
		ii =  cv_int32_get(cv);
		reti = range_check(ii, cv1, cv2, int32);
		break;
	    case CGV_DEC64: /* XXX look at fraction-digit? */
	    case CGV_INT64:
		ii =  cv_int64_get(cv);
		reti = range_check(ii, cv1, cv2, int64);
		break;
	    case CGV_UINT8:
		uu =  cv_uint8_get(cv);
		retu = range_check(uu, cv1, cv2, uint8);
		break;
	    case CGV_UINT16:
		uu =  cv_uint16_get(cv);
		retu = range_check(uu, cv1, cv2, uint16);
		break;
	    case CGV_UINT32:
		uu =  cv_uint32_get(cv);
		retu = range_check(uu, cv1, cv2, uint32);
		break;
	    case CGV_UINT64:
		uu =  cv_uint32_get(cv);
		retu = range_check(uu, cv1, cv2, uint64);
		break;
	    case CGV_STRING:
	    case CGV_REST:
		if ((str = cv_string_get(cv)) == NULL)
		    uu = 0; /* equal no string with empty string for range check */
		else
		    uu = strlen(str);
		rets = range_check(uu, cv1, cv2, uint64);
		break;
	    default:
		break;
	    }
	    /* Error handling for signed and unsigned, strings in switch
	     * OK: if check OK
	     * Failure: check fails and it is the last
	     */
	    if ((reti==0 && retu==0 && rets == 0))
		goto step1ok;
	    /* Check fails */
	    if (i==cvec_len(cvv)){ /* And it is last */
		if (reason){
		    if (reti || retu){
			if (outofrange(cv, cvv, reason) < 0)
			    goto done;
		    }
		    else
			if (outoflength(uu, cvv, reason) < 0)
			    goto done;
		}
		goto fail;
	    }
	} /* while i<cvec_len(cvv) */
    }
 step1ok:
    /* then check options for others */
    switch (cvtype){
    case CGV_STRING:
    case CGV_REST:
	str = cv_string_get(cv);
	/* Note, if there is no value, eg <s/>, str is NULL.
	 */
	if (restype){
	    if (strcmp(restype, "enumeration") == 0){
		found = 0;
		yi = NULL;
		if (str != NULL) {
		    //		    str = clixon_trim2(str, " \t\n"); /* May be misplaced, strip earlier? */
		    while ((yi = yn_each(yrestype, yi)) != NULL){
			if (yang_keyword_get(yi) != Y_ENUM)
			    continue;
			if (strcmp(yang_argument_get(yi), str) == 0){
			    found++;
			    break;
			}
		    }
		}
		if (!found){
		    if (reason)
			*reason = cligen_reason("'%s' does not match enumeration", str);
		    goto fail;
		}
	    }
	    if (strcmp(restype, "bits") == 0 && str != NULL){
		/* The lexical representation of the bits type is a space-separated list
		 * of the names of the bits that are set.  A zero-length string thus
		 * represents a value where no bits are set.
		 */
		str = clixon_trim2(str, " \t\n"); /* May be misplaced, strip earlier? */
		nvec = 0;
		if ((vec = clicon_strsep(str, " \t", &nvec)) == NULL)
		    goto done;
		for (i=0; i<nvec; i++){
		    if ((v = vec[i]) == NULL || !strlen(v))
			continue;
		    found = 0;
		    yi = NULL;
		    while ((yi = yn_each(yrestype, yi)) != NULL){
			if (yang_keyword_get(yi) != Y_BIT)
			    continue;
			if (strcmp(yang_argument_get(yi), v) == 0){
			    found++;
			    break;
			}
		    }
		    if (!found){
			if (reason)
			    *reason = cligen_reason("'%s' does not match enumeration", v);
			goto fail;
			break;
		    }
		}
	    }
	}
	if (regexps && cvec_len(regexps)) {
	    if ((ret = cv_validate_pattern(h, regexps, yrestype, str, reason)) < 0)
		goto done;
	    if (ret == 0)
		goto fail;
	}
	break;
    case CGV_VOID:
	break; /* empty type OK */
    case CGV_ERR:
	retval = 0;
	if (reason)
	    *reason = cligen_reason("Invalid cv");
	goto fail;
	break;
    default:
	break;
    }
    retval = 1; /* validation OK */
 done:
    if (vec)
	free(vec);
    return retval;
 fail:
    retval = 0; /* validation failed */
    goto done;
}

/* Forward */
static int ys_cv_validate_union(clicon_handle h,yang_stmt *ys, char **reason,
				yang_stmt *yrestype, char *type, char *val);

/*!
 * @param[out] reason  If given and return val is 0, contains a malloced string
 * @retval -1  Error (fatal), with errno set to indicate error
 * @retval 0   Validation not OK, malloced reason is returned. Free reason with free()
 * @retval 1   Validation OK
 */
static int
ys_cv_validate_union_one(clicon_handle h,
			 yang_stmt    *ys,
			 char        **reason,
			 yang_stmt    *yt,
			 char         *type,  /* orig type */
			 char         *val)
{
    int          retval = -1;
    yang_stmt   *yrt;      /* union subtype */
    int          options = 0;
    cvec        *cvv = NULL;
    cvec        *regexps = NULL;
    cvec        *patterns = NULL;
    uint8_t      fraction = 0;
    char        *restype;
    enum cv_type cvtype;
    cg_var      *cvt=NULL;

    if ((regexps = cvec_new(0)) == NULL){
	clicon_err(OE_UNIX, errno, "cvec_new");
	goto done;
    }
    if ((patterns = cvec_new(0)) == NULL){
	clicon_err(OE_UNIX, errno, "cvec_new");
	goto done;
    }
    if (yang_type_resolve(ys, ys, yt, &yrt, &options, &cvv, patterns, regexps,
			  &fraction) < 0)
	goto done;
    restype = yrt?yang_argument_get(yrt):NULL;
    if (restype && strcmp(restype, "union") == 0){      /* recursive union */
	if ((retval = ys_cv_validate_union(h, ys, reason, yrt, type, val)) < 0)
	    goto done;
    }
    else {
	if (clicon_type2cv(type, restype, ys, &cvtype) < 0)
	    goto done;
	/* reparse value with the new type */
	if ((cvt = cv_new(cvtype)) == NULL){
	    clicon_err(OE_UNIX, errno, "cv_new");
	    goto done;
	}
	if (val == NULL){ /* Fail validation on NULL */
	    retval = 0;
	    goto done;
	}
	if ((retval = cv_parse1(val, cvt, reason)) < 0){
	    clicon_err(OE_UNIX, errno, "cv_parse");
	    goto done;
	}
	if (retval == 0)
	    goto done;
	/* The regexp cache may be invalidated, in that case re-compile
	 * eg due to copying
	 */
	if (cvec_len(patterns)!=0 && cvec_len(regexps)==0){
	    if (compile_pattern2regexp(h, patterns, regexps) < 1)
		goto done;
	    if (yang_type_cache_regexp_set(yt,
					   clicon_yang_regexp(h),
					   regexps) < 0)
		goto done;
	}
	if ((retval = cv_validate1(h, cvt, cvtype, options, cvv,
				   regexps, yrt, restype, reason)) < 0)
	    goto done;
    }
 done:
    if (patterns)
	cvec_free(patterns);
    if (regexps)
	cvec_free(regexps);
    if (cvt)
	cv_free(cvt);
    return retval;
}

/*! Validate union
 * @param[out] reason  If given, and return value is 0, contains malloced string
 * @retval -1  Error (fatal), with errno set to indicate error
 * @retval 0   Validation not OK, malloced reason is returned. Free reason with free()
 * @retval 1   Validation OK
 */
static int
ys_cv_validate_union(clicon_handle h,
		     yang_stmt *ys,
		     char     **reason,
		     yang_stmt *yrestype,
		     char      *type,  /* orig type */
		     char      *val)
{
    int        retval = 1; /* valid */
    yang_stmt *yt = NULL;
    char      *reason1 = NULL;  /* saved reason */

    while ((yt = yn_each(yrestype, yt)) != NULL){
	if (yang_keyword_get(yt) != Y_TYPE)
	    continue;
	if ((retval = ys_cv_validate_union_one(h, ys, reason, yt, type, val)) < 0)
	    goto done;
	/* If validation failed, save reason, reset error and continue,
	 * save latest reason if noithing validates.
	 */
	if (retval == 0 && reason && *reason != NULL){
	    if (reason1)
		free(reason1);
	    reason1 = *reason;
	    *reason = NULL;
	}
	if (retval == 1) /* Enough that one type validates value */
	    break;
    }
 done:
    if (retval == 0 && reason1){
	*reason = reason1;
	reason1 = NULL;
    }
    if (reason1)
	free(reason1);
    return retval;
}

/*! Validate cligen variable cv using yang statement as spec
 *
 * @param[in]  h       Clicon handle
 * @param[in]  cv      A cligen variable to validate. This is a correctly parsed cv.
 * @param[in]  ys      A yang statement, must be leaf or leaf-list.
 * @param[out] reason  If given, and if return value is 0, contains malloced
 *                     string describing reason why validation failed.
 * @retval -1  Error (fatal), with errno set to indicate error
 * @retval 0   Validation not OK, malloced reason is returned. Free reason with free()
 * @retval 1   Validation OK
 * See also cv_validate - the code is similar.
 * @note reason if given must be freed by caller
 */
int
ys_cv_validate(clicon_handle h,
	       cg_var       *cv,
	       yang_stmt    *ys,
	       char        **reason)
{
    int             retval = -1;
    cg_var         *ycv;        /* cv of yang-statement */
    int             options = 0;
    cvec           *cvv = NULL;
    cvec           *patterns = NULL;
    cvec           *regexps = NULL;
    enum cv_type    cvtype;
    char           *origtype = NULL;  /* orig type */
    yang_stmt      *yrestype; /* resolved type */
    char           *restype;
    uint8_t         fraction = 0;
    int             retval2;
    char           *val;
    cg_var         *cvt=NULL;

    if (reason)
	*reason=NULL;
    if (yang_keyword_get(ys) != Y_LEAF && yang_keyword_get(ys) != Y_LEAF_LIST){
	retval = 1;
	goto done;
    }
    ycv = yang_cv_get(ys);
    if ((regexps = cvec_new(0)) == NULL){
	clicon_err(OE_UNIX, errno, "cvec_new");
	goto done;
    }
    if ((patterns = cvec_new(0)) == NULL){
	clicon_err(OE_UNIX, errno, "cvec_new");
	goto done;
    }
    if (yang_type_get(ys, &origtype, &yrestype,
		      &options, &cvv,
		      patterns, regexps,
		      &fraction) < 0)
	goto done;
    restype = yrestype?yang_argument_get(yrestype):NULL;
    if (clicon_type2cv(origtype, restype, ys, &cvtype) < 0)
	goto done;

    if (cv_type_get(ycv) != cvtype){
	/* special case: dbkey has rest syntax-> cv but yang cant have that */
	if (cvtype == CGV_STRING && cv_type_get(ycv) == CGV_REST)
	    ;
	else {
	    clicon_err(OE_DB, 0, "Type mismatch data:%s != yang:%s",
		       cv_type2str(cvtype), cv_type2str(cv_type_get(ycv)));
	    goto done;
	}
    }
    /* Note restype can be NULL here for example with unresolved hardcoded uuid */
    if (restype && strcmp(restype, "union") == 0){
	assert(cvtype == CGV_REST);
	/* Instead of NULL, give an empty string to validate, this is to avoid cv_parse
	 * errors and may actually be the wrong thing to do.
	 */
	if ((val = cv_string_get(cv)) == NULL)
	    val = "";
	if ((retval2 = ys_cv_validate_union(h, ys, reason, yrestype, origtype, val)) < 0)
	    goto done;
	retval = retval2; /* invalid (0) with latest reason or valid 1 */
    }
    else{
	/* The regexp cache may be invalidated, in that case re-compile
	 * eg due to copying
	 */
	if (cvec_len(patterns)!=0 && cvec_len(regexps)==0){
	    yang_stmt      *yt;
	    if (compile_pattern2regexp(h, patterns, regexps) < 1)
		goto done;
	    yt = yang_find(ys, Y_TYPE, NULL);
	    if (yang_type_cache_regexp_set(yt,
					   clicon_yang_regexp(h),
					   regexps) < 0)
		goto done;
	}
	if ((retval = cv_validate1(h, cv, cvtype, options, cvv,
				   regexps, yrestype, restype, reason)) < 0)
	    goto done;
    }
  done:
    if (origtype)
	free(origtype);
    if (regexps)
	cvec_free(regexps);
    if (patterns)
	cvec_free(patterns);
    if (cvt)
	cv_free(cvt);
    return retval;
}

/*
 * a typedef can be under module, submodule, container, list, grouping, rpc,
 * input, output, notification
 */
static inline int
ys_typedef(yang_stmt *ys)
{
    return yang_keyword_get(ys) == Y_MODULE || yang_keyword_get(ys) == Y_SUBMODULE ||
	yang_keyword_get(ys) == Y_CONTAINER || yang_keyword_get(ys) == Y_LIST;
}

/* find next ys up which can contain a typedef */
static yang_stmt *
ys_typedef_up(yang_stmt *ys)
{
    yang_stmt *yn;

    while (ys != NULL && !ys_typedef(ys)){
	yn = yang_parent_get(ys);
	/* Some extra stuff to ensure ys is a stmt */
	if (yn && yang_keyword_get(yn) == Y_SPEC)
	    yn = NULL;
	ys = (yang_stmt*)yn;
    }
    /* Here it is either NULL or is a typedef-kind yang-stmt */
    return (yang_stmt*)ys;
}

/*! Find identity yang-stmt given a name and a yang statement for prefix context
 *
 * @param[in] ys        Yang spec of id statement
 * @param[in] identity  Identity string on the form <prefix>:<id>
 * @retval    yid       yang-stmt of type IDENTITY
 * @retval    NULL      Not found
 * @see validate_identityref for (2) above
 * @see xml_find_identity
 */
yang_stmt *
yang_find_identity(yang_stmt *ys,
		   char      *identity)
{
    char        *id = NULL;
    char        *prefix = NULL;
    yang_stmt   *ymodule;
    yang_stmt   *yid = NULL;
    yang_stmt   *yn;

    if (nodeid_split(identity, &prefix, &id) < 0)
	goto done;
    /* No, now check if identityref is derived from base */
    if (prefix){ /* Go to top and find import that matches */
	if ((ymodule = yang_find_module_by_prefix(ys, prefix)) == NULL)
	    goto done;
	/* if ymodule is a sub-module, the identity may be found in a
	 * sub-module of ymod */
	yid = yang_find(ymodule, Y_IDENTITY, id);
    }
    else{
	while (1){
	    /* Check upwards in hierarchy for matching typedefs */
	    if ((ys = ys_typedef_up(ys)) == NULL) /* If reach top */
		break;
	    /* Here find identity */
	    if ((yid = yang_find(ys, Y_IDENTITY, id)) != NULL)
		break;
	    /* Did not find a matching typedef there, proceed to next level */
	    yn = yang_parent_get(ys);
	    if (yn && yang_keyword_get(yn) == Y_SPEC)
		yn = NULL;
	    ys = (yang_stmt*)yn;
	}
    }
  done:
    if (id)
	free(id);
    if (prefix)
	free(prefix);
    return yid;
}

/*! Find identity yang-stmt given a name and a xml node for prefix context
 *
 * @param[in] yspec     Top-level yang-spec
 * @param[in] identity  Identity string on the form <prefix>:<id>
 * @param[in] nsc       Namespace context for <prefix>
 * @retval    yid       yang-stmt of type IDENTITY
 * @retval    NULL      Not found
 * @see validate_identityref for (2) above
 * @see xml_find_identity
 */
yang_stmt *
yang_find_identity_nsc(yang_stmt *yspec,
		       char      *identity,
		       cvec      *nsc)
{
    char        *id = NULL;
    char        *prefix = NULL;
    yang_stmt   *ymodule;
    yang_stmt   *yid = NULL;
    char        *ns = NULL;

    if (nodeid_split(identity, &prefix, &id) < 0)
	goto done;
    if ((ns = xml_nsctx_get(nsc, prefix)) == NULL)
	goto done;
    if ((ymodule = yang_find_module_by_namespace(yspec, ns)) == NULL)
	goto done;
    /* if ymodule is a sub-module, the identity may be found in a
     * sub-module of ymod */
    yid = yang_find(ymodule, Y_IDENTITY, id);
  done:
    if (id)
	free(id);
    if (prefix)
	free(prefix);
    return yid;
}

/*! Resolve type restrictions, return constraining parameters
 *
 * This is for types with range/length/regexp restrictions of the base type
 * Also fraction-digits for decimal64 is handled as that.
 * @param[in]  ytype    yang-stmt object containing currently resolving type
 * @param[out] options   Pointer to flags field of optional values. optional
 * @param[out] cvv       Pointer to cvec with min range or length.
 *                       If options&YANG_OPTIONS_RANGE or YANG_OPTIONS_LENGTH
 * @param[out] regexps   Pointer to cvec of compiled patterns
 * @param[out] fraction  For decimal64, how many digits after period
 * @retval    -1         Error
 * @retval     0         OK.
 */
static int
yang_type_resolve_restrictions(yang_stmt   *ytype,
			       int         *options,
			       cvec       **cvv,
			       cvec        *regexps,
			       uint8_t     *fraction)
{
    int        retval = -1;
    yang_stmt *ys;
    cg_var    *cv;
    char      *pattern;

    if (options && cvv &&
	(ys = yang_find(ytype, Y_RANGE, NULL)) != NULL){
	*cvv = yang_cvec_get(ys);
	*options  |= YANG_OPTIONS_RANGE;
    }
    if (options && cvv &&
	(ys = yang_find(ytype, Y_LENGTH, NULL)) != NULL){
	*cvv = yang_cvec_get(ys);
	*options  |= YANG_OPTIONS_LENGTH;
    }
    /* Find all patterns */
    if (options && regexps){
	ys = NULL;
	while ((ys = yn_each(ytype, ys)) != NULL) {
	    if (yang_keyword_get(ys) != Y_PATTERN)
		continue;
	    if ((cv = cvec_add(regexps, CGV_STRING)) == NULL){
		clicon_err(OE_UNIX, errno, "cvec_add");
		goto done;
	    }
	    pattern = yang_argument_get(ys); /* clear text pattern */
	    /* Check 1.1 invert pattern */
	    if (yang_find(ys, Y_MODIFIER, "invert-match") != NULL)
		cv_flag_set(cv, V_INVERT);
	    cv_string_set(cv, pattern);
	}
    }
    if (options && fraction &&
	(ys = yang_find(ytype, Y_FRACTION_DIGITS, NULL)) != NULL){
	*fraction  = cv_uint8_get(yang_cv_get(ys));
	*options  |= YANG_OPTIONS_FRACTION_DIGITS;
    }
    retval = 0;
 done:
    return retval;
}

/*! Recursively resolve a yang type to built-in type with optional restrictions
 * @param[in]  yorig    (original) type yang-stmt where original search is based
 * @param[in]  ys       (transitive) yang-stmt where current search is based
 * @param[in]  ytype    yang-stmt object containing currently resolving type
 * @param[out] yrestype Resolved type. return built-in type or NULL.
 * @param[out] options  Flags field of optional values, see YANG_OPTIONS_*
 * @param[out] cvv      Cvec with min/max range or length.
 *                      Present if options&YANG_OPTIONS_RANGE|_LENGTH.
 *                      Can be a vector if multiple ranges
 * @param[out] patterns Initialized cvec of regexp patterns strings (if any)
 * @param[out] regexps  Initialized cvec of compiled regexps (if any)
 * @param[out] fraction for decimal64, how many digits after period
 *                      Present if options&YANG_OPTIONS_FRACTION_DIGITS
 * @retval      0        OK. Note yrestype may still be NULL.
 * @retval     -1        Error, clicon_err handles errors
 * The setting of the options argument has the following semantics:
 *   options&YANG_OPTIONS_RANGE or YANG_OPTIONS_LENGTH --> cvv is set containing
 *                   array of range_min, range_max cv:s
 *   options&YANG_OPTIONS_FRACTION_DIGITS --> fraction is set
 *   patterns && cvec_len(patterns) --> there are patterns
 * Note that the static output strings (type, pattern) should be copied if used asap.
 * Note also that for all pointer arguments, if NULL is given, no value is assigned.
 */
int
yang_type_resolve(yang_stmt   *yorig,
		  yang_stmt   *ys,
		  yang_stmt   *ytype,
		  yang_stmt  **yrestype,
		  int         *options,
		  cvec       **cvv,
		  cvec        *patterns,
		  cvec        *regexps,
		  uint8_t     *fraction)
{
    yang_stmt   *rytypedef = NULL; /* Resolved typedef of ytype */
    yang_stmt   *rytype;           /* Resolved type of ytype */
    char        *type = NULL;
    char        *prefix = NULL;
    int          retval = -1;
    yang_stmt   *yn;
    yang_stmt   *yrmod; /* module where resolved type is looked for */
    int          ret;

    if (options)
	*options = 0x0;
    *yrestype    = NULL; /* Initialization of resolved type that may not be necessary */

    if (nodeid_split(yang_argument_get(ytype), &prefix, &type) < 0)
	goto done;
    /* Cache does not work for eg string length 32? */
#if 1
    if ((ret = yang_type_cache_get(ytype, yrestype,
				   options, cvv, patterns, NULL, regexps, fraction)) < 0)
	goto done;
    if (ret == 1)
	goto ok;
#else
    if (ytype->ys_typecache != NULL){
	if (yang_type_cache_get(ytype, yrestype,
				options, cvv, patterns, NULL, regexps, fraction) < 0)
	    goto done;
	goto ok;
    }
#endif
    /* Check if type is basic type. If so, return that */
    if ((prefix == NULL && yang_builtin(type))){
	*yrestype = ytype;
	if (yang_type_resolve_restrictions(ytype, options, cvv, patterns, fraction) < 0)
	    goto done;
	goto ok;
    }

    /* Not basic type. Now check if prefix which means we look in other module */
    if (prefix){ /* Go to top and find import that matches */
	if ((yrmod = yang_find_module_by_prefix(ytype, prefix)) == NULL){
	    clicon_err(OE_DB, 0, "Type not resolved: \"%s:%s\" in module %s",
		       prefix, type, yang_argument_get(ys_module(yorig)));
	    goto done;
	}
	if ((rytypedef = yang_find(yrmod, Y_TYPEDEF, type)) == NULL)
	    goto ok; /* unresolved */
	ys = rytypedef;
    }
    else
	while (1){
	    /* Check upwards in hierarchy for matching typedefs */
	    if ((ys = ys_typedef_up(ys)) == NULL){ /* If reach top */
		*yrestype = NULL;
		break;
	    }
	    /* Here find typedef */
	    if ((rytypedef = yang_find(ys, Y_TYPEDEF, type)) != NULL)
		break;
	    /* Did not find a matching typedef there, proceed to next level */
	    yn = yang_parent_get(ys);
	    if (yn && (yang_keyword_get(yn) == Y_SPEC))
		yn = NULL;
	    ys = (yang_stmt*)yn;
	}
    if (rytypedef != NULL){     /* We have found a typedef */
	/* Find associated type statement */
	if ((rytype = yang_find(rytypedef, Y_TYPE, NULL)) == NULL){
	    clicon_err(OE_DB, 0, "mandatory type object is not found");
	    goto done;
	}
	/* recursively resolve this new type */
	if (yang_type_resolve(yorig, ys, rytype, yrestype,
			      options, cvv,
			      patterns, regexps,
			      fraction) < 0)
	    goto done;
	/* appends patterns, overwrites others if any */
	if (yang_type_resolve_restrictions(ytype, options, cvv, patterns, fraction) < 0)
	    goto done;
    }
  ok:
    retval = 0;
  done:
    if (prefix)
	free(prefix);
    if (type)
	free(type);
    return retval;
}

/*! Get type information about a leaf/leaf-list yang-statement
 *
 * @code
 *   yang_stmt    *yrestype;
 *   char         *origtype = NULL;
 *   int           options;
 *   cvec         *cvv = NULL;
 *   cvec         *patterns = cvec_new(0);
 *   cvec         *regexps = cvec_new(0);
 *   uint8_t       fraction;
 *
 *   if (yang_type_get(ys, &origtype, &yrestype, &options, &cvv,
 *                     patterns, regexps, &fraction) < 0)
 *      goto err;
 *   if (yrestype == NULL) # unresolved
 *      goto err;
 *   if (options & YANG_OPTIONS_LENGTH != 0)
 *      printf("%d..%d\n", min , max);
 * @endcode
 * @param[in]  ys       yang-stmt, leaf or leaf-list
 * @param[out] origtype original type may be derived or built-in (malloced)
 * @param[out] yrestype Resolved type. return built-in type or NULL.
 * @param[out] options  Flags field of optional values, see YANG_OPTIONS_*
 * @param[out] cvv      Cvec with min/max range or length.
 *                      Present if options&YANG_OPTIONS_RANGE|_LENGTH.
 *                      Can be a vector if multiple ranges
 * @param[out] pattern  yang cvec pattern POSIX regexp patterns
 * @param[out] regexps  Initialized cvec of compiled regexps (if any)
 * @param[out] fraction for decimal64, how many digits after period
 *                      Present if options&YANG_OPTIONS_FRACTION_DIGITS
 * @retval      0       OK, but note that restype==NULL means not resolved.
 * @retval     -1       Error, clicon_err handles errors
 * The setting of the options argument has the following semantics:
 *   options&YANG_OPTIONS_RANGE or YANG_OPTIONS_LENGTH --> cvv is set containing
 *                   array of range_min, range_max cv:s
 *   options&YANG_OPTIONS_FRACTION_DIGITS --> fraction is set
 * Note that the static output strings (type, pattern) should be copied if used asap.
 * Note also that for all pointer arguments, if NULL is given, no value is assigned.
 * @See yang_type_resolve(). This function is really just a frontend to that.
 */
int
yang_type_get(yang_stmt    *ys,
	      char        **origtype,
	      yang_stmt   **yrestype,
	      int          *options,
	      cvec        **cvv,
	      cvec         *patterns,
	      cvec         *regexps,
	      uint8_t      *fraction
    )
{
    int retval = -1;
    yang_stmt    *ytype;        /* type */
    char         *type = NULL;

    if (options)
	*options = 0x0;
    /* Find mandatory type */
    if ((ytype = yang_find(ys, Y_TYPE, NULL)) == NULL){
	clicon_err(OE_DB, ENOENT, "mandatory type object is not found");
	goto done;
    }
    /* XXX: here we seem to have some problems if type is union */
    if (nodeid_split(yang_argument_get(ytype), NULL, &type) < 0)
	goto done;
    if (origtype &&
	(*origtype = strdup(type)) == NULL){
	clicon_err(OE_XML, errno, "stdup");
	goto done;
    }
    if (yang_type_resolve(ys, ys, ytype, yrestype,
			  options, cvv, patterns, regexps, fraction) < 0)
	goto done;
    retval = 0;
  done:
    if (type)
	free(type);
    return retval;
}

/*! Utility function to translate a leaf/leaf-list to its base CV-type only
 * @see yang_type_get  Full leaf/list type api
 */
enum cv_type
yang_type2cv(yang_stmt  *ys)
{
    yang_stmt      *yrestype;  /* resolved type */
    char           *restype;  /* resolved type */
    char           *origtype=NULL;   /* original type */
    enum cv_type    cvtype = CGV_ERR;

    /* Find type specification */
    if (yang_type_get(ys, &origtype, &yrestype, NULL, NULL, NULL, NULL, NULL)
 < 0)
	goto done;
    restype = yrestype?yang_argument_get(yrestype):NULL;
    if (clicon_type2cv(origtype, restype, ys, &cvtype) < 0) /* This handles non-resolved also */
	goto done;
 done:
    if (origtype)
	free(origtype);
    return cvtype;
}