customdumpdata.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245
  1. # Licensed to Cloudera, Inc. under one
  2. # or more contributor license agreements. See the NOTICE file
  3. # distributed with this work for additional information
  4. # regarding copyright ownership. Cloudera, Inc. licenses this file
  5. # to you under the Apache License, Version 2.0 (the
  6. # "License"); you may not use this file except in compliance
  7. # with the License. You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. from django.core.exceptions import ImproperlyConfigured
  17. from django.core.management.base import NoArgsCommand, CommandError, OutputWrapper
  18. from django.core import serializers
  19. from django.db import router, DEFAULT_DB_ALIAS
  20. from django.utils.datastructures import SortedDict
  21. from django.contrib.auth.models import User
  22. from optparse import make_option
  23. import logging
  24. import sys
  25. #class Command(BaseCommand):
  26. # option_list = BaseCommand.option_list + (
  27. # make_option('--format', default='json', dest='format',
  28. # help='Specifies the output serialization format for fixtures.'),
  29. # make_option('--indent', default=None, dest='indent', type='int',
  30. # help='Specifies the indent level to use when pretty-printing output'),
  31. # make_option('--database', action='store', dest='database',
  32. # default=DEFAULT_DB_ALIAS, help='Nominates a specific database to dump '
  33. # 'fixtures from. Defaults to the "default" database.'),
  34. # make_option('-e', '--exclude', dest='exclude',action='append', default=[],
  35. # help='An appname or appname.ModelName to exclude (use multiple --exclude to exclude multiple apps/models).'),
  36. # make_option('-n', '--natural', action='store_true', dest='use_natural_keys', default=False,
  37. # help='Yay CHRIS Use natural keys if they are available.'),
  38. # make_option('-a', '--all', action='store_true', dest='use_base_manager', default=False,
  39. # help="Use Django's base manager to dump all models stored in the database, including those that would otherwise be filtered or modified by a custom manager."),
  40. # make_option('--pks', dest='primary_keys', help="Only dump objects with "
  41. # "given primary keys. Accepts a comma seperated list of keys. "
  42. # "This option will only work when you specify one model."),
  43. # )
  44. # help = ("Output the contents of the database as a fixture of the given "
  45. # "format (using each model's default manager unless --all is "
  46. # "specified).")
  47. # args = '[appname appname.ModelName ...]'
  48. # def handle(self, *app_labels, **options):
  49. class Command(NoArgsCommand):
  50. def handle_noargs(self, *app_labels, **options):
  51. from django.db.models import get_app, get_apps, get_model
  52. format = options.get('format')
  53. indent = options.get('indent')
  54. using = options.get('database')
  55. excludes = options.get('exclude')
  56. show_traceback = options.get('traceback')
  57. use_natural_keys = options.get('use_natural_keys')
  58. use_base_manager = options.get('use_base_manager')
  59. pks = options.get('primary_keys')
  60. user = options.get('user')
  61. userid = user.id
  62. stdout = OutputWrapper(options.get('stdout', sys.stdout))
  63. if pks:
  64. primary_keys = pks.split(',')
  65. else:
  66. primary_keys = []
  67. excluded_apps = set()
  68. excluded_models = set()
  69. if excludes:
  70. for exclude in excludes:
  71. if '.' in exclude:
  72. app_label, model_name = exclude.split('.', 1)
  73. model_obj = get_model(app_label, model_name)
  74. if not model_obj:
  75. raise CommandError('Unknown model in excludes: %s' % exclude)
  76. excluded_models.add(model_obj)
  77. else:
  78. try:
  79. app_obj = get_app(exclude)
  80. excluded_apps.add(app_obj)
  81. except ImproperlyConfigured:
  82. raise CommandError('Unknown app in excludes: %s' % exclude)
  83. if len(app_labels) == 0:
  84. if primary_keys:
  85. raise CommandError("You can only use --pks option with one model")
  86. app_list = SortedDict((app, None) for app in get_apps() if app not in excluded_apps)
  87. else:
  88. if len(app_labels) > 1 and primary_keys:
  89. raise CommandError("You can only use --pks option with one model")
  90. app_list = SortedDict()
  91. for label in app_labels:
  92. try:
  93. app_label, model_label = label.split('.')
  94. try:
  95. app = get_app(app_label)
  96. except ImproperlyConfigured:
  97. raise CommandError("Unknown application: %s" % app_label)
  98. if app in excluded_apps:
  99. continue
  100. model = get_model(app_label, model_label)
  101. if model is None:
  102. raise CommandError("Unknown model: %s.%s" % (app_label, model_label))
  103. if app in app_list.keys():
  104. if app_list[app] and model not in app_list[app]:
  105. app_list[app].append(model)
  106. else:
  107. app_list[app] = [model]
  108. except ValueError:
  109. if primary_keys:
  110. raise CommandError("You can only use --pks option with one model")
  111. # This is just an app - no model qualifier
  112. app_label = label
  113. try:
  114. app = get_app(app_label)
  115. except ImproperlyConfigured:
  116. raise CommandError("Unknown application: %s" % app_label)
  117. if app in excluded_apps:
  118. continue
  119. app_list[app] = None
  120. # Check that the serialization format exists; this is a shortcut to
  121. # avoid collating all the objects and _then_ failing.
  122. if format not in serializers.get_public_serializer_formats():
  123. try:
  124. serializers.get_serializer(format)
  125. except serializers.SerializerDoesNotExist:
  126. pass
  127. raise CommandError("Unknown serialization format: %s" % format)
  128. def get_objects():
  129. # Collate the objects to be serialized.
  130. for model in sort_dependencies(app_list.items()):
  131. if model in excluded_models:
  132. continue
  133. if not model._meta.proxy and router.allow_syncdb(using, model):
  134. if use_base_manager:
  135. objects = model._base_manager
  136. else:
  137. objects = model._default_manager
  138. queryset = objects.using(using).order_by(model._meta.pk.name)
  139. if primary_keys:
  140. queryset = queryset.filter(pk__in=primary_keys)
  141. queryset = queryset.filter(owner_id=userid)
  142. for obj in queryset.iterator():
  143. yield obj
  144. try:
  145. stdout.ending = None
  146. # self.stdout.ending = None
  147. serializers.serialize(format, get_objects(), indent=indent,
  148. use_natural_keys=use_natural_keys, stream=stdout)
  149. stdout
  150. # serializers.serialize(format, get_objects(), indent=indent,
  151. # use_natural_keys=use_natural_keys, stream=self.stdout)
  152. except Exception as e:
  153. if show_traceback:
  154. raise
  155. raise CommandError("Unable to serialize database: %s" % e)
  156. def sort_dependencies(app_list):
  157. """Sort a list of app,modellist pairs into a single list of models.
  158. The single list of models is sorted so that any model with a natural key
  159. is serialized before a normal model, and any model with a natural key
  160. dependency has it's dependencies serialized first.
  161. """
  162. from django.db.models import get_model, get_models
  163. # Process the list of models, and get the list of dependencies
  164. model_dependencies = []
  165. models = set()
  166. for app, model_list in app_list:
  167. if model_list is None:
  168. model_list = get_models(app)
  169. for model in model_list:
  170. models.add(model)
  171. # Add any explicitly defined dependencies
  172. if hasattr(model, 'natural_key'):
  173. deps = getattr(model.natural_key, 'dependencies', [])
  174. if deps:
  175. deps = [get_model(*d.split('.')) for d in deps]
  176. else:
  177. deps = []
  178. # Now add a dependency for any FK or M2M relation with
  179. # a model that defines a natural key
  180. for field in model._meta.fields:
  181. if hasattr(field.rel, 'to'):
  182. rel_model = field.rel.to
  183. if hasattr(rel_model, 'natural_key') and rel_model != model:
  184. deps.append(rel_model)
  185. for field in model._meta.many_to_many:
  186. rel_model = field.rel.to
  187. if hasattr(rel_model, 'natural_key') and rel_model != model:
  188. deps.append(rel_model)
  189. model_dependencies.append((model, deps))
  190. model_dependencies.reverse()
  191. # Now sort the models to ensure that dependencies are met. This
  192. # is done by repeatedly iterating over the input list of models.
  193. # If all the dependencies of a given model are in the final list,
  194. # that model is promoted to the end of the final list. This process
  195. # continues until the input list is empty, or we do a full iteration
  196. # over the input models without promoting a model to the final list.
  197. # If we do a full iteration without a promotion, that means there are
  198. # circular dependencies in the list.
  199. model_list = []
  200. while model_dependencies:
  201. skipped = []
  202. changed = False
  203. while model_dependencies:
  204. model, deps = model_dependencies.pop()
  205. # If all of the models in the dependency list are either already
  206. # on the final model list, or not on the original serialization list,
  207. # then we've found another model with all it's dependencies satisfied.
  208. found = True
  209. for candidate in ((d not in models or d in model_list) for d in deps):
  210. if not candidate:
  211. found = False
  212. if found:
  213. model_list.append(model)
  214. changed = True
  215. else:
  216. skipped.append((model, deps))
  217. if not changed:
  218. raise CommandError("Can't resolve dependencies for %s in serialized app list." %
  219. ', '.join('%s.%s' % (model._meta.app_label, model._meta.object_name)
  220. for model, deps in sorted(skipped, key=lambda obj: obj[0].__name__))
  221. )
  222. model_dependencies = skipped
  223. return model_list