Эх сурвалжийг харах

[notebook] Provides a Github API client to fetch file contents for file in a public Github repo

Example(s):

/notebook/api/github/fetch?url=https://github.com/ipython/ipython/blob/1.x/examples/notebooks/Part%203%20-%20Plotting%20with%20Matplotlib.ipynb
/notebook/api/github/fetch?url=https://github.com/mqlaql/geospatial-data/blob/master/Geospatial-Data-with-Python.ipynb
Jenny Kim 10 жил өмнө
parent
commit
98784ed

+ 20 - 0
desktop/libs/notebook/src/notebook/api.py

@@ -27,6 +27,7 @@ from desktop.models import Document2, Document
 
 from notebook.connectors.base import get_api, Notebook, QueryExpired
 from notebook.decorators import api_error_handler, check_document_modify_permission
+from notebook.github import GithubClient, GithubClientException
 from notebook.models import escape_rows
 
 
@@ -296,3 +297,22 @@ def autocomplete(request, database=None, table=None, column=None, nested=None):
   response['status'] = 0
 
   return JsonResponse(response)
+
+
+@api_error_handler
+def github_fetch(request):
+  response = {'status': -1}
+
+  api = GithubClient()
+
+  response['url'] = url = request.GET.get('url')
+
+  if url:
+    owner, repo, branch, filepath = api.parse_github_url(url)
+
+    response['status'] = 0
+    response['content'] = api.get_file_contents(owner, repo, filepath, branch)
+  else:
+    response['message'] = _('fetch_github requires full URL to Github file.')
+
+  return JsonResponse(response)

+ 148 - 0
desktop/libs/notebook/src/notebook/github.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python
+# -- coding: utf-8 --
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import binascii
+import json
+import logging
+import re
+import urllib
+
+from desktop.lib.rest.http_client import HttpClient, RestException
+from desktop.lib.rest import resource
+
+
+LOG = logging.getLogger(__name__)
+
+
+class GithubClientException(Exception):
+  pass
+
+
+class GithubClient(object):
+  """
+  https://developer.github.com/v3/
+  """
+
+  BASE_URL = 'https://api.github.com'
+
+  REPO_URL_RE = re.compile("http[s]?://(www.)?github.com/([a-z0-9](?:-?[a-z0-9]){0,38}).([\w\.@\:\-~]+)/blob/([\w\.@\:\-~_]+)/(.+)?")
+
+
+  def __init__(self, **options):
+    # TODO: Add support for access token and authenticated API access
+    self._client = HttpClient(self.BASE_URL, logger=LOG)
+    self._root = resource.Resource(self._client)
+
+
+  @classmethod
+  def parse_github_url(cls, url):
+    """
+    Given a base URL to a Github repository, return a tuple of the owner, repo, branch, and filepath
+    :param url: base URL to repo (e.g. - https://github.com/cloudera/hue/blob/master/README.rst)
+    :return: tuple of strings (e.g. - ('cloudera', 'hue', 'master', 'README.rst'))
+    """
+    match = cls.REPO_URL_RE.search(url)
+    if match:
+      return match.group(2), match.group(3), match.group(4), match.group(5)
+    else:
+      raise ValueError('Github URL is not formatted correctly: %s' % url)
+
+
+  def _get_headers(self):
+    return {}
+
+
+  def _get_params(self):
+    return ()
+
+
+  def _get_json(self, response):
+    if type(response) != dict:
+      # Got 'plain/text' mimetype instead of 'application/json'
+      try:
+        response = json.loads(response)
+      except ValueError, e:
+        # Got some null bytes in the response
+        LOG.error('%s: %s' % (unicode(e), repr(response)))
+        response = json.loads(response.replace('\x00', ''))
+    return response
+
+
+  def _clean_path(self, filepath):
+    cleaned_path = filepath.strip('/')
+    cleaned_path = urllib.unquote(cleaned_path)
+    return cleaned_path
+
+
+  def get_file_contents(self, owner, repo, filepath, branch='master'):
+    filepath = self._clean_path(filepath)
+    try:
+      sha = self.get_sha(owner, repo, filepath, branch)
+      blob = self.get_blob(owner, repo, sha)
+      content = blob['content'].decode('base64')
+      return content
+    except binascii.Error:
+      raise GithubClientException('Failed to decode file contents, check if file content is properly base64-encoded.')
+    except GithubClientException, e:
+      raise e
+    except Exception, e:
+      raise GithubClientException('Failed to get file contents: %s' % str(e))
+
+
+  def get_sha(self, owner, repo, filepath, branch='master'):
+    filepath = self._clean_path(filepath)
+    try:
+      sha = branch
+      path_tokens = filepath.split('/')
+      for token in path_tokens:
+        tree = self.get_tree(owner, repo, sha, recursive=False)
+        sha = next(elem['sha'] for elem in tree['tree'] if elem['path'] == token)
+      return sha
+    except StopIteration, e:
+      raise GithubClientException('Could not find sha for: %s/%s/%s/%s' % (owner, repo, branch, filepath))
+    except RestException, e:
+      raise e
+
+
+  def get_tree(self, owner, repo, sha='master', recursive=True):
+    """
+    GET /repos/:owner/:repo/git/trees/:sha
+    https://developer.github.com/v3/git/trees/#get-a-tree
+    """
+    try:
+      params = self._get_params()
+      if recursive:
+        params += (
+            ('recursive', 1),
+        )
+      response = self._root.get('repos/%s/%s/git/trees/%s' % (owner, repo, sha), headers=self._get_headers(), params=params)
+      return self._get_json(response)
+    except RestException, e:
+      raise GithubClientException('Could not find Github object, check owner, repo and filepath or permissions: %s' % str(e))
+
+
+  def get_blob(self, owner, repo, sha):
+    """
+    GET /repos/:owner/:repo/git/blobs/:sha
+    https://developer.github.com/v3/git/blobs/#get-a-blob
+    """
+    try:
+      response = self._root.get('repos/%s/%s/git/blobs/%s' % (owner, repo, sha), headers=self._get_headers(), params=self._get_params())
+      return self._get_json(response)
+    except RestException, e:
+      raise GithubClientException('Could not find Github object, check owner, repo and sha or permissions: %s' % str(e))

+ 6 - 1
desktop/libs/notebook/src/notebook/urls.py

@@ -60,6 +60,11 @@ urlpatterns += patterns('notebook.api',
   url(r'^api/notebook/close$', 'close_notebook', name='close_notebook'),
 )
 
+# Github
+urlpatterns += patterns('notebook.api',
+  url(r'^api/github/fetch$', 'github_fetch', name='github_fetch'),
+)
+
 # Assist API
 urlpatterns += patterns('notebook.api',
   url(r'^api/autocomplete/$', 'autocomplete', name='api_autocomplete_databases'),
@@ -70,4 +75,4 @@ urlpatterns += patterns('notebook.api',
   url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)$', 'autocomplete', name='api_autocomplete_column'),
   url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)/$', 'autocomplete', name='api_autocomplete_column'),
   url(r'^api/autocomplete/(?P<database>\w+)/(?P<table>\w+)/(?P<column>\w+)/(?P<nested>.+)$', 'autocomplete', name='api_autocomplete_nested'),
-)
+)