api3.py 3.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. import json
  18. import logging
  19. from django.utils.translation import ugettext as _
  20. from desktop.lib.django_util import JsonResponse
  21. from desktop.lib.exceptions_renderable import PopupException
  22. from libsolr.api import SolrApi
  23. from search.conf import SOLR_URL, SECURITY_ENABLED
  24. from indexer.controller2 import IndexController
  25. from indexer.utils import get_default_fields
  26. from hadoop import cluster
  27. from indexer.smart_indexer import Indexer
  28. from indexer.controller import CollectionManagerController
  29. LOG = logging.getLogger(__name__)
  30. def _escape_white_space_characters(s, inverse = False):
  31. MAPPINGS = {
  32. "\n":"\\n",
  33. "\t":"\\t",
  34. "\r":"\\r",
  35. " ":"\\s"
  36. }
  37. to = 1 if inverse else 0
  38. from_ = 0 if inverse else 1
  39. for pair in MAPPINGS.iteritems():
  40. s = s.replace(pair[to], pair[from_]).encode('utf-8')
  41. return s
  42. def _convert_format(format_dict, inverse=False):
  43. for field in format_dict:
  44. if isinstance(format_dict[field], basestring):
  45. format_dict[field] = _escape_white_space_characters(format_dict[field], inverse)
  46. def guess_format(request):
  47. file_format = json.loads(request.POST.get('fileFormat', '{}'))
  48. indexer = Indexer(request.user, request.fs)
  49. stream = request.fs.open(file_format["path"])
  50. format_ = indexer.guess_format({
  51. "file":{
  52. "stream":stream,
  53. "name":file_format['path']
  54. }
  55. })
  56. _convert_format(format_)
  57. return JsonResponse(format_)
  58. def guess_field_types(request):
  59. file_format = json.loads(request.POST.get('fileFormat', '{}'))
  60. indexer = Indexer(request.user, request.fs)
  61. stream = request.fs.open(file_format["path"])
  62. _convert_format(file_format["format"], inverse = True)
  63. format_ = indexer.guess_field_types({
  64. "file":{
  65. "stream":stream,
  66. "name":file_format['path']
  67. },
  68. "format":file_format['format']
  69. })
  70. return JsonResponse(format_)
  71. def index_file(request):
  72. file_format = json.loads(request.POST.get('fileFormat', '{}'))
  73. _convert_format(file_format["format"], inverse = True)
  74. collection_name = file_format["name"]
  75. indexer = Indexer(request.user, request.fs)
  76. unique_field = indexer.get_unique_field(file_format)
  77. is_unique_generated = indexer.is_unique_generated(file_format)
  78. schema_fields = indexer.get_kept_field_list(file_format['columns'])
  79. if is_unique_generated:
  80. schema_fields += [{"name": unique_field, "type": "string"}]
  81. morphline = indexer.generate_morphline_config(collection_name, file_format, unique_field)
  82. collection_manager = CollectionManagerController(request.user)
  83. if not collection_manager.collection_exists(collection_name):
  84. collection_manager.create_collection(collection_name, schema_fields, unique_key_field=unique_field)
  85. job_id = indexer.run_morphline(collection_name, morphline, file_format["path"])
  86. return JsonResponse({"jobId": job_id})