doc2_utils.py 3.7 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697
  1. # Licensed to Cloudera, Inc. under one
  2. # or more contributor license agreements. See the NOTICE file
  3. # distributed with this work for additional information
  4. # regarding copyright ownership. Cloudera, Inc. licenses this file
  5. # to you under the Apache License, Version 2.0 (the
  6. # "License"); you may not use this file except in compliance
  7. # with the License. You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. import json
  17. import re
  18. import logging
  19. import difflib
  20. from desktop.models import Document2
  21. #DOC2_NAME_INVALID_CHARS = "[<>/{}[\]~`u'\xe9'u'\xfa'u'\xf3'u'\xf1'u'\xed']"
  22. DOC2_NAME_INVALID_CHARS = "[<>/{}\[\]]"
  23. LOG = logging.getLogger(__name__)
  24. def removeInvalidChars(fixString):
  25. fixString = re.sub(r'[^\x00-\x7f]',r'', fixString)
  26. return re.sub(DOC2_NAME_INVALID_CHARS, '', fixString)
  27. def findMatchingQuery(user, id, name, query, include_history=False, all=False, values=False):
  28. #Returns list of matching queries. If all = False
  29. #returns at first found for speed
  30. name = removeInvalidChars(name)
  31. # LOG.info("finding queries that match name: %s" % name)
  32. documents = getSavedQueries(user=user, name=name, include_history=include_history)
  33. matchdocs = []
  34. matchvalues = []
  35. for doc in documents:
  36. if all == True or not matchdocs:
  37. matchdata = json.loads(doc.data)
  38. matchname = removeInvalidChars(doc.name)
  39. # LOG.info("found name: matchname: %s" % matchname)
  40. if 'snippets' in matchdata:
  41. matchquery = matchdata['snippets'][0]['statement_raw']
  42. if re.match(name, matchname) and id != doc.id:
  43. # LOG.info("Query name: %s and matchname: %s are similar" % (name, matchname))
  44. # LOG.info("Comparing queries:")
  45. if query == matchquery:
  46. # LOG.info("MATCHED QUERY: name: %s: id: %s" % (name, id))
  47. matchdocs.append(doc)
  48. matchvalues.append(doc.id)
  49. if values == False:
  50. # LOG.info("returning %s matching docs" % len(matchdocs))
  51. return matchdocs
  52. else:
  53. # LOG.info("returning %s matching doc ids" % len(matchdocs))
  54. return matchvalues
  55. def getSavedQueries(user, name=None, include_history=False):
  56. #mimic api call to get saved queries
  57. perms = 'both'
  58. include_trashed = False
  59. flatten = True
  60. if name:
  61. # LOG.info("getting queries that match name: %s" % name)
  62. if include_history:
  63. documents = Document2.objects.filter(name__contains=name, owner=user, type__in=['query-hive', 'query-impala'])
  64. # documents = Document2.objects.filter(name__iregex=r'%s.*' %name, owner=user, type__in=['query-hive', 'query-impala'])
  65. else:
  66. documents = Document2.objects.filter(name__contains=name, owner=user, type__in=['query-hive', 'query-impala'], is_history=include_history)
  67. # documents = Document2.objects.filter(name__iregex=r'%s.*' %name, owner=user, type__in=['query-hive', 'query-impala'], is_history=include_history)
  68. else:
  69. # LOG.info("getting all queries")
  70. if include_history:
  71. documents = Document2.objects.filter(
  72. user=user,
  73. include_trashed=include_trashed,
  74. type__in=['query-hive', 'query-impala']
  75. )
  76. else:
  77. documents = Document2.objects.filter(
  78. user=user,
  79. include_history=include_history,
  80. include_trashed=include_trashed,
  81. type__in=['query-hive', 'query-impala']
  82. )
  83. # LOG.info("returning queries, total count: %s" % len(documents))
  84. return documents