#!/usr/bin/python3

# parse arguments before importing the modules because networkx has a very slow
# startup time
import argparse
parser = argparse.ArgumentParser(description="Extract queries from FILE, parse"
        " them against the database schema specified in SCHEMA and write them"
        " to OUTFILE in a format that is suitable to be read by the Rad"
        " framework.  FILE can either be a PHP file or a page XML file.")

parser.add_argument("pickle_file", metavar="SCHEMA", help="path to a schema"
        " file generated by genschema.py using the --pickle-out option")
parser.add_argument("json_file", metavar="OUTFILE", help="if the file exists,"
        " the newly parsed queries are merged into the list of queries already"
        " present in OUTFILE; the data is serialized using JSON and can"
        " optionally be indented with the --indent switch")
parser.add_argument("input_file", nargs="+", metavar="PATH", help=
        "path to one or more PHP or XML files")
parser.add_argument("--indent", dest="indent_json", action="store_true",
        help="indent JSON output and sort its dictionary keys (off by default)")
parser.set_defaults(indent_json=False)

args = parser.parse_args()

import collections
import hashlib
import itertools
import json
import networkx as nx
import os.path
import pickle
import sqlparse
import sys
import types

import rad.console
from rad.parser import init_parser, match_token, peek, consume, take
from rad.sql import *
from lxml import etree
from phply import phplex, phpparse, phpast

RAD_NAMESPACE = "http://cbikt.de/rad-framework/page"
RAD = "{" + RAD_NAMESPACE + "}"
SERIALIZATION_FORMAT_VERSION = 5
T = sqlparse.tokens.Token

########################################
############  QUERY PARSER  ############
########################################
class ColumnSpec:
    def __init__(self, table, name):
        self.table = table
        self.name = name
    def to_tuple(self):
        return (self.table, self.name)
    def __repr__(self):
        return "Column {}.{}".format(self.table, self.name)
    def to_sql(self):
        return "`{}`.`{}`".format(self.table, self.name)

class Clause:
    def __init__(self, operand1, operator, operand2):
        self.operand1 = operand1
        self.operator = operator
        self.operand2 = operand2

    def __repr__(self):
        return "Clause {} {} {}".format(self.operand1, self.operator, self.operand2)

    def to_sql(self):
        return "{} {} {}".format(self.operand1.to_sql(),
                self.operator, self.operand2.to_sql())

class Conditions(set):
    def to_sql(self):
        sql = []
        for clause in self:
            if isinstance(clause, Clause):
                sql.append(clause.to_sql())
            else:
                sql.append("(" + clause + ")")
        return " AND ".join(sql)

class SelectExpression:
    def __init__(self, column, alias):
        self.column = column
        self.alias = alias

    def to_sql(self):
        return "{} AS `{}`".format(self.column.to_sql(), self.alias)

class TableSpec:
    def __init__(self, name, alias):
        self.name = name
        self.alias = alias

    def to_sql(self):
        return "`{}` `{}`".format(self.name, self.alias)

class Join:
    def __init__(self, join_type, table, conditions):
        self.join_type = join_type
        self.table = table
        self.conditions = conditions

    def to_sql(self):
        sql = []
        if self.join_type == "inner":
            sql.append("INNER JOIN")
        elif self.join_type == "left":
            sql.append("LEFT JOIN")
        else:
            assert False
        sql.append(self.table.to_sql())
        if self.conditions is not None:
            sql.append("ON")
            sql.append("(")
            sql.append(self.conditions.to_sql())
            sql.append(")")

        return " ".join(sql)

class Query:
    def __init__(self):
        self.columns = []
        self.table = None
        self.joins = []
        self.where = None
        self.order_by = []
        self.aliases = None
        self.is_distinct = False

    def set_distinct(self, distinct=True):
        self.is_distinct = distinct

    def add_select_expression(self, expr):
        self.columns.append(expr)

    def set_table(self, tablespec):
        self.table = tablespec

    def set_where(self, conditions):
        self.where = conditions

    def add_join(self, join):
        self.joins.append(join)

    def set_aliases(self, aliases):
        self.aliases = aliases

    def add_order_by(self, col, order):
        self.order_by.append((col, order))

    def foreach_clause(self, callback):
        def visit_conditions(conditions):
            for cond in conditions:
                if isinstance(cond, Clause):
                    callback(cond)
        for join in self.joins:
            if join.conditions is not None:
                visit_conditions(join.conditions)
        if self.where is not None:
            visit_conditions(self.where)

    def to_sql(self):
        return {
            "select": self._to_sql_select(),
            "from": self._to_sql_from(),
            "where": self._to_sql_where(),
            "order_by": self._to_sql_order_by(),
        }

    def _to_sql_select(self):
        sql = []
        sql.append("SELECT")
        if self.is_distinct:
            sql.append("DISTINCT")
        cols = []
        for expr in self.columns:
            cols.append(expr.to_sql())
        sql.append(", ".join(cols))
        return " ".join(sql)

    def _to_sql_from(self):
        sql = []
        sql.append("FROM")
        sql.append(self.table.to_sql())
        for join in self.joins:
            sql.append(join.to_sql())
        return " ".join(sql)

    def _to_sql_where(self):
        if self.where is not None:
            return "WHERE " + self.where.to_sql()
        else:
            return ""

    def _to_sql_order_by(self):
        if len(self.order_by):
            parts = []
            for (col, order) in self.order_by:
                parts.append("{} {}".format(col.to_sql(),
                    order if order is not None else ""))
            return "ORDER BY " + ", ".join(parts)
        else:
            return ""

def parse_query(sql):
    # various grammar constructs
    def parse_alias():
        if peek(T.Keyword, "AS"):
            consume()
            return get_name(consume(T.Name))
        elif peek(T.Name):
            return get_name(consume())
        else:
            return None

    def parse_column():
        col = get_name(consume(T.Name))
        if peek(T.Punctuation, "."):
            consume()
            table = col
            col = get_name(consume(T.Name))
        else:
            table = None
        return ColumnSpec(table, col)

    def parse_select_expr():
        col = parse_column()
        alias = parse_alias()
        return SelectExpression(col, alias)

    def parse_table():
        table = get_name(consume(T.Name))
        alias = parse_alias()
        return TableSpec(table, alias)

    def parse_clause():
        def parse_subcondition():
            level = 0
            subcond = []
            while True:
                if peek(T.Punctuation, ")"):
                    if level == 0:
                        break
                tok = consume()
                if match_token(tok, T.Punctuation, "("):
                    level += 1
                elif match_token(tok, T.Punctuation, ")"):
                    level -= 1
                subcond.append(tok.value)
            return " ".join(subcond)
        def parse_operand():
            if peek(T.Name):
                return parse_column()
            elif peek(T.Literal):
                val = parse_sql_value(allow_null=True)
                return val
        def parse_operator():
            return consume(T.Operator, "=")

        if peek(T.Punctuation, "("):
            consume()
            cond = parse_subcondition()
            consume(T.Punctuation, ")")
            return cond
        elif peek(T.Name):
            a = parse_column()
            op = parse_operator()
            b = parse_operand()
            return Clause(a, op.value, b)
        elif peek(T.Literal):
            a = parse_operand()
            op = parse_operator()
            b = parse_column()
            return Clause(a, op.value, b)
        else:
            raise ValueError("Conditional clause expected.")

    def parse_conditions():
        conds = Conditions()
        while True:
            conds.add(parse_clause())
            if not peek(T.Keyword, "AND"):
                break
            consume()
        return conds

    def parse_join():
        if peek(T.Keyword, "INNER JOIN"):
            consume()
            join_type = "inner"
        #elif peek(T.Keyword, "LEFT JOIN"):
        #    consume()
        #    join_type = "left"
        else:
            return None
        table = parse_table()
        if peek(T.Keyword, "ON"):
            consume()
            if peek(T.Punctuation, "("):
                consume() # remove outer layer of parentheses
                cond = parse_conditions()
                consume(T.Punctuation, ")")
            else:
                cond = parse_conditions()
        else:
            cond = None
        return Join(join_type, table, cond)

    # main
    #print("Parsing query {}...".format(sql))
    stmt = sqlparse.parse(sql)
    if len(stmt) > 1 or not isinstance(stmt[0], sqlparse.sql.Statement):
        sys.stderr.write("Must be a single SQL statement.\n")
        sys.exit(1)
    stmt = stmt[0]
    init_parser(stmt)

    query = Query()
    consume(T.Keyword.DML, "SELECT")
    if peek(T.Keyword, "DISTINCT"):
        consume()
        query.is_distinct = True
    while True:
        expr = parse_select_expr()
        query.add_select_expression(expr)
        if not peek(T.Punctuation, ","):
            break
        consume()
    consume(T.Keyword, "FROM")
    query.set_table(parse_table())
    while True:
        join = parse_join()
        if join is None:
            break
        query.add_join(join)
    if peek(T.Keyword, "WHERE"):
        consume()
        query.set_where(parse_conditions())
    if peek(T.Keyword, "ORDER"):
        consume()
        consume(T.Keyword, "BY")
        while True:
            col = parse_column()
            if peek(T.Keyword.Order):
                tok = consume()
                order = tok.value
            else:
                order = None
            query.add_order_by(col, order)
            if not peek(T.Punctuation, ","):
                break
            consume() # consume comma
    if peek(T.Keyword, "ORDER BY"):
        consume()
        while True:
            col = parse_column()
            if peek(T.Keyword.Order):
                tok = consume()
                order = tok.value
            else:
                order = None
            query.add_order_by(col, order)
            if not peek(T.Punctuation, ","):
                break
            consume() # consume comma
    t = take()
    if t is not None:
        raise ValueError("Unexpected additional bytes: "+t)

    return query

def query_semantic_analysis(query, schema):
    def search_column(cname):
        tables = []
        for al in aliases:
            if schema[aliases[al]].has_column(cname):
                tables.append(al)
        return tables

    def complete_column(col):
        if col.table is None:
            all_tables = search_column(col.name)
            if len(all_tables) == 0:
                raise ValueError("Unknown column '{}'.".format(col.name))
            if len(all_tables) > 1:
                raise ValueError("Ambiguous column reference '{}'."
                        .format(col.name))
            col.table = all_tables[0]
        if not col.table in aliases:
            raise ValueError("Column '{}' refers to unknown table '{}'."
                    .format(col.name, col.table))
        if not schema[aliases[col.table]].has_column(col.name):
            raise ValueError("Table '{}' has no column '{}'."
                    .format(col.table, col.name))

    def complete_clause(clause):
        if isinstance(clause.operand1, ColumnSpec):
            complete_column(clause.operand1)
        if isinstance(clause.operand2, ColumnSpec):
            complete_column(clause.operand2)
    aliases = {}

    # fill in table aliases...
    # ... in FROM clause:
    if not query.table.name in schema:
        raise ValueError("Unknown table '{}' in FROM clause."
                .format(query.table.name))
    if query.table.alias is None:
        query.table.alias = query.table.name
    aliases[query.table.alias] = query.table.name
    # ... in JOIN clauses:
    for join in query.joins:
        if not join.table.name in schema:
            raise ValueError("Unknown table '{}' in JOIN clause."
                    .format(join.table.name))
        if join.table.alias is None:
            join.table.alias = join.table.name
        if join.table.alias in aliases:
            raise ValueError("Ambiguous table alias '{}' in JOIN."
                    .format(table[1]))
        aliases[join.table.alias] = join.table.name

    # complete column references
    for expr in query.columns:
        complete_column(expr.column)
    query.foreach_clause(complete_clause)
    for (col, _) in query.order_by:
        complete_column(col)

    # complete column aliases
    column_aliases = set()
    for expr in query.columns:
        if expr.alias is None:
            expr.alias = expr.column.name
        if expr.alias in column_aliases:
            raise ValueError("Ambiguous alias '{}' in SELECT clause."
                    .format(expr.alias))
        column_aliases.add(expr.alias)

    query.set_aliases(aliases)


def preprocess_query(query, schema):
    def colset_to_dict(colset):
        return {"{}.{}".format(table, col): True for (table, col) in colset}

    def columns_by_table(columns):
        ret = collections.defaultdict(list)
        for (table, col) in columns:
            ret[table].append(col)
        return ret

    def generate_update_graph(query, schema):
        assert query.aliases is not None
        G = nx.MultiDiGraph()
        G.add_nodes_from(query.aliases.keys())

        # construct dependency graph
        def walk_conditions(conds):
            for clause in conds:
                if not isinstance(clause, Clause):
                    continue
                op1 = clause.operand1
                op2 = clause.operand2
                if not (isinstance(op1, ColumnSpec) and isinstance(op2, ColumnSpec)):
                    continue
                if op1.table == op2.table:
                    continue
                G.add_edge(op1.table, op2.table, key=(op1.name, op2.name))
                G.add_edge(op2.table, op1.table, key=(op2.name, op1.name))
        for join in query.joins:
            if join.conditions is not None:
                walk_conditions(join.conditions)
        if query.where is not None:
            walk_conditions(query.where)

        return G

    def generate_insert_graph(G_update, query, schema):
        G = nx.MultiDiGraph()
        G.add_nodes_from(G_update.nodes())
        ro_columns = set()

        for (t1, t2, key) in G_update.edges(keys=True):
            (c1, c2) = key
            table1 = schema[query.aliases[t1]]
            table2 = schema[query.aliases[t2]]
            c1_auto = table1.columns[c1].flags & Column.FLAG_AUTO
            c2_auto = table2.columns[c2].flags & Column.FLAG_AUTO
            if c1_auto and c2_auto:
                rad.console.error(file=sys.stderr)
                sys.stderr.write("Cannot join on two AUTO columns.\n")
                sys.exit(2)
            elif c1_auto:
                ro_columns.add((t1, c1))
                G.add_edge(t1, t2, key=(c1, c2))
            elif c2_auto:
                ro_columns.add((t2, c2))
                G.add_edge(t2, t1, key=(c2, c1))

        return G, ro_columns

    def reduce_graph(G_orig, masters):
        #print("Reducing from masters: " + unicode(masters))
        def remove_edge(t1, t2, key):
            #print("Removing {}->{} ({})".format(t1, t2, key))
            G.remove_edge(t1, t2, key=key)
            (c, _) = key
            ro_columns.add((t1, c))

        A = set(masters)
        G = G_orig.copy()
        ro_columns = set()

        # remove edges within A:
        for (t1, t2, key) in list(G.edges(A, keys=True)):
            if t1 in A and t2 in A:
                remove_edge(t1, t2, key)
        # I: all edges pointing into A
        I = [(t1, t2, key) for (t1, t2, key) in list(G.in_edges(A, keys=True))
                           if t1 not in A]
        # O: list of nodes reachable in one step from A
        O = collections.defaultdict(set)
        for (_, t2, (c1, c2)) in G.out_edges(A, keys=True):
            if t2 not in A:
                O[t2].add(c2)

        while len(A) < G.order():
            # remove all edges pointing into A
            for (t1, t2, key) in I:
                remove_edge(t1, t2, key)
            # pick a node v reachable in one step from A
            if len(O) == 0:
                #print("Not all nodes are reachable from master set.")
                return None, None
            for (v, cols) in O.items():
                if schema[query.aliases[v]].is_unique(cols):
                    del O[v]
                    break
            else:
                #print("Graph is not weakly connected.")
                return None, None
            # add v to A
            A.add(v)
            # update O
            for (_, t2, (c1, c2)) in  list(G.out_edges([v], keys=True)):
                O[t2].add(c2)
            # add all ingoing edges to v that are not in A to I
            I = [(t1, t2, key) for (t1, t2, key) in list(G.in_edges([v], keys=True))
                               if t1 not in A]
        return G, ro_columns

    def find_best_reduced_graph(G, seed):
        N = set(G.nodes())
        N.remove(seed)

        best_graph = None
        best_masters = None
        best_ro_columns = None
        best_edge_count = -1
        for r in range(0, len(N) + 1):
            for subset in itertools.combinations(N, r):
                masters = frozenset(subset + (seed,))

                G_local, ro_columns = reduce_graph(G, masters)
                if G_local is None:
                    continue
                if G_local.size() > best_edge_count:
                    best_graph = G_local
                    best_edge_count = G_local.size()
                    best_masters = masters
                    best_ro_columns = ro_columns
        return best_graph, best_masters, best_ro_columns

    all_columns = set()
    ro_columns = set()
    for select_expr in query.columns:
        all_columns.add(select_expr.column.to_tuple())

    # dependency graph in update order
    G = generate_update_graph(query, schema)

    # dependency graph in insert order
    G_insert, ro_columns_insert = generate_insert_graph(G, query, schema)
    ro_columns.update(ro_columns_insert)

    # find best reduced version of G
    G, masters, ro_columns_reduced = find_best_reduced_graph(G, query.table.alias)
    if G is None:
        rad.console.error(file=sys.stderr)
        sys.stderr.write("Error: Could not resolve all dependencies within query.\n")
        sys.exit(2)
    ro_columns.update(ro_columns_reduced)

    # add all dependent fields to SELECT list
    for (t1, t2, (c1, c2)) in list(G.edges(keys=True)):
        all_columns.add((t1, c1))
        all_columns.add((t2, c2))

    # add all primary keys to SELECT list
    for (alias, table) in query.aliases.items():
        for pkcol in schema[table].primary_key:
            all_columns.add((alias, pkcol))

    # make all AUTO columns that appear in the SELECT clause read-only:
    for select_expr in query.columns:
        col = select_expr.column
        table = schema[query.aliases[col.table]]
        if table.columns[col.name].flags & Column.FLAG_AUTO:
            ro_columns.add((col.table, col.name))
    #print("Read-only columns: {}".format(ro_columns))

    qdict = {}

    # extract binds
    def extract_binds(clause):
        if not isinstance(clause.operand1, ColumnSpec):
            column, literal = clause.operand2.get_value(), clause.operand1
        elif not isinstance(clause.operand2, ColumnSpec):
            column, literal = clause.operand1, clause.operand2.get_value()
        else:
            return # a clause with two literals as operands should pass the
                   # parser anyway
        if isinstance(literal, ColumnSpec):
            return # not interested
        query_binds["{}.{}".format(column.table, column.name)] = literal
    query_binds = {}
    query.foreach_clause(extract_binds)
    qdict["binds"] = query_binds

    # column<->alias map and its inverse:
    col_by_alias = {}
    alias_by_col = {}
    for select_expr in query.columns:
        (table, col) = select_expr.column.to_tuple()
        col_by_alias[select_expr.alias] = (table, col)
        alias_by_col["{}.{}".format(table, col)] = select_expr.alias
    qdict["col_by_alias"] = col_by_alias
    qdict["alias_by_col"] = alias_by_col

    # in-edges of the update graph:
    in_edges = collections.defaultdict(set)
    for (t1, t2, (c1, c2)) in list(G.edges(keys=True)):
        in_edges[t2].add(((t1, c1), (t2, c2)))
    qdict["in_edges_update"] = in_edges

    # out-edges of the insert graph
    out_edges = collections.defaultdict(set)
    for (t1, t2, (c1, c2)) in list(G_insert.edges(keys=True)):
        out_edges[t1].add((t2, c2))
    qdict["out_edges_insert"] = out_edges

    # find all NEW-able columns
    newable_columns = set()
    for (t1, t2, (c1, c2)) in list(G.edges(keys=True)):
        table2 = schema[query.aliases[t2]]
        if table2.columns[c2].flags & Column.FLAG_AUTO:
            newable_columns.add((t1, c1))
    qdict["newable_columns"] = colset_to_dict(newable_columns)

    qdict["columns_by_table"] = columns_by_table(all_columns)
    qdict["ro_columns"] = colset_to_dict(ro_columns)
    qdict["tables_update"] = nx.topological_sort(G)
    qdict["tables_insert"] = nx.topological_sort(G_insert)
    qdict["primary_table"] = query.table.alias
    qdict["masters"] = masters

    # overwrite SELECT list before generating SQL query:
    query.columns = []
    for (table, col) in all_columns:
        alias = "{}.{}".format(table, col)
        query.add_select_expression(SelectExpression(ColumnSpec(table, col), alias))
    sql_parts = query.to_sql()
    qdict["sql_select"] = sql_parts["select"]
    qdict["sql_from"] = sql_parts["from"]
    qdict["sql_where"] = sql_parts["where"]
    qdict["sql_order_by"] = sql_parts["order_by"]

    # convert colspecs used in ORDER BY clause into column aliases
    order_by_aliases = collections.OrderedDict()
    for (column, order) in query.order_by:
        (table, col) = column.to_tuple()
        colspec = "{}.{}".format(table, col)
        if colspec in alias_by_col:
            alias = alias_by_col[colspec]
            if alias not in order_by_aliases:
                order_by_aliases[alias] = (alias, order != "DESC")
        else:
            rad.console.warn()
            print("Ordering by column that is not selected: {}".format(colspec))
    qdict["order_by_aliases"] = order_by_aliases

    return qdict


########################################
########   PHP AND XML PARSER   ########
########################################
def extract_queries_php(path):
    def visit(node):
        if isinstance(node, phpast.StaticMethodCall):
            if node.name != "get":
                return
            cls_name = node.class_
        elif isinstance(node, phpast.New):
            cls_name = node.name
        else:
            return

        if not isinstance(cls_name, str):
            return
        cls_name = cls_name.split("\\")[-1]
        if cls_name not in ("RecordSet", "Query"):
            return
        if len(node.params) < 1:
            return
        first_param = node.params[0]
        if first_param.is_ref:
            return
        if not isinstance(first_param.node, str):
            return
        queries.append(first_param.node.decode("utf-8"))

    if extract_queries_php.parser is None:
        extract_queries_php.parser = phpparse.make_parser()
    queries = []
    with open(path, "r") as f:
        instr = f.read()
    lexer = phplex.lexer.clone()
    lexer.filename = path
    ast = extract_queries_php.parser.parse(input=instr, lexer=lexer)
    for node in ast:
        node.accept(visit)
    return queries
extract_queries_php.parser = None

def extract_queries_xml(path):
    if extract_queries_xml.parser is None:
        extract_queries_xml.parser = etree.XMLParser(remove_comments=True)
    parser = extract_queries_xml.parser
    tree = etree.parse(path, parser=parser)
    for el in tree.findall("{}model//{}sql".format(RAD, RAD)):
        yield el.text
extract_queries_xml.parser = None


########################################
############      MAIN      ############
########################################
# load schema:
with open(args.pickle_file, "rb") as f:
    try:
        (schema, schema_classes, format_version) = pickle.load(f)
    except:
        rad.console.error(file=sys.stderr)
        sys.stderr.write("Could not read schema from '{}':\n"
                .format(args.pickle_file))
        (e_type, e_value, _) = sys.exc_info()
        sys.stderr.write("{}: {}\n".format(e_type.__name__, e_value))
        sys.exit(3)
    if format_version != SERIALIZATION_FORMAT_VERSION:
        rad.console.error(file=sys.stderr)
        sys.stderr.write("The schema in '{}' was generated with an incompatible"
                " version of genschema.py, aborting.\n".format(args.pickle_file))
        sys.exit(4)

# parse queries:
queries = {}
seen_hashes = set()

for path in args.input_file:
    ext = os.path.splitext(path)[1].lower()
    if ext == ".php":
        raw_queries = extract_queries_php(path)
    elif ext == ".xml":
        raw_queries = extract_queries_xml(path)
    else:
        rad.console.error(file=sys.stderr)
        sys.stderr.write("Path '{}' must have .php or .xml extension.\n"
                .format(path))
        sys.exit(2)

    for sql in raw_queries:
        sha = hashlib.sha256(sql.encode("utf-8")).hexdigest()
        if sha in seen_hashes:
            continue
        seen_hashes.add(sha)
        try:
            query = parse_query(sql)
        except ValueError as e:
            rad.console.error()
            print("Couldn't parse query, ignoring (error was: {}).".format(e))
            print("'"+sql+"'")
            print(e)
            sys.exit(5)
        query_semantic_analysis(query, schema)
        qdict = preprocess_query(query, schema)
        qdict["original_query"] = sql
        qdict["table_classes"] = {alias: schema_classes[table]
                                  for (alias, table) in query.aliases.items()}
        queries[sha] = qdict

# generate json file:
if os.path.exists(args.json_file):
    print("Reading existing queries from {}...".format(args.json_file))
    try:
        with open(args.json_file, "r") as f:
            queries_existing = json.load(f, encoding="utf-8")
    except:
        rad.console.error(file=sys.stderr)
        (e_type, e_value, _) = sys.exc_info()
        sys.stderr.write("{}: {}\n".format(e_type.__name__, e_value))
        sys.exit(6)
    queries_existing.update(queries)
    queries = queries_existing

print("Generating file {}...".format(args.json_file))
class SetToListEncoder(json.JSONEncoder):
    def default(self, o):
        #print(type(o))

        if isinstance(o, (set, frozenset, types.GeneratorType)):
            return list(o)
        else:
            return json.JSONEncoder.default(self, o)
with open(args.json_file, "w", encoding='utf8') as f:
    q1 = queries
    #print(queries)
    s = ""
    if args.indent_json:

        s = json.dumps(q1, cls=SetToListEncoder, indent=True, sort_keys=True)
    else:
        s = json.dumps(q1, cls=SetToListEncoder)
    f.write(s)