pig_setup.py 2.4 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. import logging
  18. import os
  19. from django.contrib.auth.models import User
  20. from django.core import management
  21. from django.core.management.base import NoArgsCommand
  22. from django.db import transaction
  23. from django.utils.translation import ugettext as _
  24. from hadoop import cluster
  25. from desktop.lib import paths
  26. from desktop.models import Document
  27. from liboozie.submittion import create_directories
  28. from pig.conf import LOCAL_SAMPLE_DIR, REMOTE_SAMPLE_DIR
  29. from useradmin.models import install_sample_user
  30. LOG = logging.getLogger(__name__)
  31. class Command(NoArgsCommand):
  32. def handle_noargs(self, **options):
  33. fs = cluster.get_hdfs()
  34. create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
  35. remote_dir = REMOTE_SAMPLE_DIR.get()
  36. # Copy examples binaries
  37. for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
  38. local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
  39. remote_data_dir = fs.join(remote_dir, name)
  40. LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
  41. 'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
  42. fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)
  43. # Copy sample data
  44. local_dir = paths.get_thirdparty_root("sample_data")
  45. remote_data_dir = fs.join(remote_dir, 'data')
  46. LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
  47. 'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
  48. fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)
  49. # Load jobs
  50. install_sample_user()
  51. management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2)
  52. Document.objects.sync()