[SEBA-450] (part 1)
Refactoring, python3 compat, and tox tests on:
- xosconfig
- xosgenx
- xosutil
Eliminate use of yaml.load() which is unsafe, switch to yaml.safe_load()
More diagnostics during database migration
Change-Id: I0fae5782fca401603a7c4e4ec2b9269ad24bda97
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/__init__.py b/lib/xos-genx/xosgenx/jinja2_extensions/__init__.py
index bf7a812..a50bf3a 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/__init__.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/__init__.py
@@ -12,6 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# Ignore this file as the * imports for Jinja2 functions are too numerous to list
+# flake8: noqa
+
+from __future__ import absolute_import
from .django import *
from .base import *
from .fol2 import *
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/base.py b/lib/xos-genx/xosgenx/jinja2_extensions/base.py
index 96e8dc2..f4929a9 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/base.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/base.py
@@ -12,8 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-from __future__ import print_function
+from __future__ import absolute_import, print_function
import pdb
import re
from inflect import engine as inflect_engine_class
@@ -319,10 +318,11 @@
def xproto_string_type(xptags):
- try:
- max_length = eval(xptags["max_length"])
- except BaseException:
- max_length = 1024
+ # FIXME: this try/except block assigns but never uses max_length?
+ # try:
+ # max_length = eval(xptags["max_length"])
+ # except BaseException:
+ # max_length = 1024
if "varchar" not in xptags:
return "string"
@@ -340,26 +340,59 @@
def xproto_field_graph_components(fields, model, tag="unique_with"):
+ """
+ NOTE: Don't use set theory operators if you want repeatable tests - many
+ of them have non-deterministic behavior
+ """
+
def find_components(graph):
+
+ # 'graph' dict structure:
+ # - keys are strings
+ # - values are sets containing strings that are names of other keys in 'graph'
+
+ # take keys from 'graph' dict and put in 'pending' set
pending = set(graph.keys())
+
+ # create an empty list named 'components'
components = []
+ # loop while 'pending' is true - while there are still items in the 'pending' set
while pending:
+
+ # remove a random item from pending set, and put in 'front'
+ # this is the primary source of nondeterminism
front = {pending.pop()}
+
+ # create an empty set named 'component'
component = set()
+ # loop while 'front' is true. Front is modified below
while front:
+
+ # take the (only?) item out of the 'front' dict, and put in 'node'
node = front.pop()
+
+ # from 'graph' dict take set with key of 'node' and put into 'neighbors'
neighbours = graph[node]
+
+ # remove the set of items in components from neighbors
neighbours -= component # These we have already visited
+
+ # add all remaining neighbors to front
front |= neighbours
+ # remove neighbors from pending
pending -= neighbours
+
+ # add neighbors to component
component |= neighbours
- components.append(component)
+ # append component set to components list, sorted
+ components.append(sorted(component))
- return components
+ # return 'components', which is a list of sets
+ return sorted(components)
field_graph = {}
field_names = {f["name"] for f in fields}
@@ -378,6 +411,7 @@
field_graph.setdefault(f["name"], set()).add(uf)
field_graph.setdefault(uf, set()).add(f["name"])
+
except KeyError:
pass
@@ -434,12 +468,12 @@
def xproto_field_to_swagger_enum(f):
if "choices" in f["options"]:
- list = []
+ c_list = []
for c in eval(xproto_unquote(f["options"]["choices"])):
- list.append(c[0])
+ c_list.append(c[0])
- return list
+ return sorted(c_list)
else:
return False
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/checklib.py b/lib/xos-genx/xosgenx/jinja2_extensions/checklib.py
index db61f01..c361b64 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/checklib.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/checklib.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
import ast
@@ -27,13 +28,13 @@
except SyntaxError:
return "511 Could not parse sync step %s" % sync_step_path
- classes = filter(lambda x: isinstance(x, ast.ClassDef), sync_step_ast.body)
+ classes = [x for x in sync_step_ast.body if isinstance(x, ast.ClassDef)]
found_sync_step_class = False
for c in classes:
base_names = [v.id for v in c.bases]
if "SyncStep" in base_names or "SyncInstanceUsingAnsible" in base_names:
- attributes = filter(lambda x: isinstance(x, ast.Assign), c.body)
+ attributes = [x for x in c.body if isinstance(x, ast.Assign)]
for a in attributes:
target_names = [t.id for t in a.targets]
values = a.value.elts if isinstance(a.value, ast.List) else [a.value]
@@ -66,7 +67,7 @@
except SyntaxError:
return "511 Could not parse sync step %s" % model_policy_path
- classes = filter(lambda x: isinstance(x, ast.ClassDef), model_policy_ast.body)
+ classes = [x for x in model_policy_ast.body if isinstance(x, ast.ClassDef)]
found_model_policy_class = False
for c in classes:
base_names = [v.id for v in c.bases]
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/django.py b/lib/xos-genx/xosgenx/jinja2_extensions/django.py
index 3909c00..0c34d3e 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/django.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/django.py
@@ -12,11 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import print_function
-from base import *
-import pdb
+from __future__ import absolute_import, print_function
+from .base import unquote
import re
import sys
+from six.moves import map
def django_content_type_string(xptags):
@@ -48,7 +48,7 @@
if "content_type" in xptags:
return django_content_type_string(xptags)
- elif max_length < 1024 * 1024:
+ elif int(max_length) < 1024 * 1024:
return "CharField"
else:
return "TextField"
@@ -208,7 +208,7 @@
else:
lst = []
- for k, v in d.items():
+ for k, v in sorted(d.items(), key=lambda t: t[0]): # sorted by key
if k in known_validators:
validator_lst.append(use_native_django_validators(k, v))
elif isinstance(v, str) and k == "default" and v.endswith('()"'):
@@ -261,7 +261,7 @@
def xproto_validations(options):
try:
return [
- map(str.strip, validation.split(":"))
+ list(map(str.strip, validation.split(":")))
for validation in unquote(options["validators"]).split(",")
]
except KeyError:
@@ -280,7 +280,7 @@
optioned_fields = []
for f in fields:
option_names = []
- for k, v in f["options"].items():
+ for k, v in sorted(f["options"].items(), key=lambda t: t[0]): # sorted by key
option_names.append(k)
if option in option_names and f["options"][option] == val:
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/fol2.py b/lib/xos-genx/xosgenx/jinja2_extensions/fol2.py
index 6d66117..0997e00 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/fol2.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/fol2.py
@@ -12,14 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-from __future__ import print_function
+from __future__ import absolute_import, print_function
import astunparse
import ast
import random
import string
import jinja2
-from plyxproto.parser import *
+from plyxproto.parser import lex, yacc
+from plyxproto.logicparser import FOLParser, FOLLexer
+from six.moves import range
+from six.moves import input
BINOPS = ["|", "&", "->"]
QUANTS = ["exists", "forall"]
@@ -45,11 +47,13 @@
self.idx = 0
return self
- def next(self):
+ def __next__(self):
var = "i%d" % self.idx
self.idx += 1
return var
+ next = __next__ # 2to3
+
def gen_random_string():
return "".join(
@@ -63,18 +67,18 @@
self.loopvar = iter(AutoVariable("i"))
self.verdictvar = iter(AutoVariable("result"))
- self.loop_variable = self.loopvar.next()
- self.verdict_variable = self.verdictvar.next()
+ self.loop_variable = next(self.loopvar)
+ self.verdict_variable = next(self.verdictvar)
self.context_map = context_map
if not self.context_map:
self.context_map = {"user": "self", "obj": "obj"}
def loop_next(self):
- self.loop_variable = self.loopvar.next()
+ self.loop_variable = next(self.loopvar)
def verdict_next(self):
- self.verdict_variable = self.verdictvar.next()
+ self.verdict_variable = next(self.verdictvar)
def gen_enumerate(self, fol):
pass
@@ -398,7 +402,8 @@
if result['result'] == 'True':
return {'hoist': ['const'], 'result': result['hoist'][1]}
elif result['hoist'][0] in BINOPS:
- return {'hoist': ['const'], 'result': {result['hoist'][0]: [result['hoist'][1], {k: [var2, result['result']]}]}}
+ return {'hoist': ['const'], 'result': {result['hoist'][0]:
+ [result['hoist'][1], {k: [var2, result['result']]}]}}
else:
return {'hoist': ['const'], 'result': {k: [var2, result['result']]}}
else:
@@ -731,9 +736,10 @@
if fol_reduced in ["True", "False"] and fol != fol_reduced:
raise TrivialPolicy(
- "Policy %(name)s trivially reduces to %(reduced)s. If this is what you want, replace its contents with %(reduced)s" % {
- "name": policy,
- "reduced": fol_reduced})
+ "Policy %(name)s trivially reduces to %(reduced)s."
+ "If this is what you want, replace its contents with %(reduced)s"
+ % {"name": policy, "reduced": fol_reduced}
+ )
a = f2p.gen_test_function(fol_reduced, policy, tag="security_check")
@@ -749,9 +755,10 @@
if fol_reduced in ["True", "False"] and fol != fol_reduced:
raise TrivialPolicy(
- "Policy %(name)s trivially reduces to %(reduced)s. If this is what you want, replace its contents with %(reduced)s" % {
- "name": policy,
- "reduced": fol_reduced})
+ "Policy %(name)s trivially reduces to %(reduced)s."
+ "If this is what you want, replace its contents with %(reduced)s"
+ % {"name": policy, "reduced": fol_reduced}
+ )
a = f2p.gen_validation_function(fol_reduced, policy, message, tag="validator")
@@ -762,7 +769,7 @@
while True:
inp = ""
while True:
- inp_line = raw_input()
+ inp_line = input()
if inp_line == "EOF":
break
else:
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/gui.py b/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
index 4cb644a..07dbc35 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/gui.py
@@ -12,8 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-from base import xproto_string_type, unquote
+from __future__ import absolute_import
+from .base import xproto_string_type, unquote
def xproto_type_to_ui_type(f):
@@ -49,6 +49,20 @@
return None
+def xproto_dict_to_sorted_string(d):
+ """
+ sorts the dict by key and returns a string representation, which makes
+ for better consistency when testing
+ """
+ ft = [] # formatted tuples
+ for k, v in sorted(d.items(), key=lambda t: t[0]): # sorted by key
+ if v is not None:
+ ft.append("'%s': '%s'" % (k, v))
+ else:
+ ft.append("'%s': None" % k)
+ return "{%s}" % ", ".join(ft)
+
+
def xproto_validators(f):
# To be cleaned up when we formalize validation in xproto
validators = []
diff --git a/lib/xos-genx/xosgenx/jinja2_extensions/tosca.py b/lib/xos-genx/xosgenx/jinja2_extensions/tosca.py
index d8eada7..4424824 100644
--- a/lib/xos-genx/xosgenx/jinja2_extensions/tosca.py
+++ b/lib/xos-genx/xosgenx/jinja2_extensions/tosca.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import absolute_import
from xosgenx.jinja2_extensions import xproto_field_graph_components
@@ -58,18 +59,19 @@
):
keys.append("%s_id" % f["name"])
# if not keys are specified and there is a name field, use that as key.
- if len(keys) == 0 and "name" in map(lambda f: f["name"], fields):
+ if len(keys) == 0 and "name" in [f["name"] for f in fields]:
keys.append("name")
- for of in one_of:
+ for of in sorted(one_of):
# check if the field is a link, and in case add _id
for index, f in enumerate(of):
try:
- field = [
+ # FIXME: the 'field' var appears to be dead code, but exists to cause the IndexError?
+ field = [ # noqa: F841
x for x in fields if x["name"] == f and ("link" in x and x["link"])
][0]
of[index] = "%s_id" % f
- except IndexError as e:
+ except IndexError:
# the field is not a link
pass