浏览代码

HUE-5788 [core] Add github api to list files in assist

Includes
1. Moving github library from desktop/notebook to desktop/lib
2. New Frontend Api to support Git tab in Assist
3. Single point API in desktop/lib/svc to support any kind of version control for the future
krish 8 年之前
父节点
当前提交
c91b7bfbba

+ 38 - 12
desktop/conf.dist/hue.ini

@@ -507,6 +507,44 @@
           # The username attribute in the LDAP schema
           ## group_name_attr=cn
 
+  # Configuration options for specifying the Source Version Control.
+  # ----------------------------------------------------------------
+  [[vcs]]
+
+  ## [[[git-read-only]]]
+      ## Base URL to Remote Server
+      # remote_url=https://github.com/cloudera/hue/tree/master
+
+      ## Base URL to Version Control API
+      # api_url=https://api.github.com
+  ## [[[github]]]
+
+      ## Base URL to Remote Server
+      # remote_url=https://github.com/cloudera/hue/tree/master
+
+      ## Base URL to Version Control API
+      # api_url=https://api.github.com
+
+      # These will be necessary when you want to write back to the repository.
+      ## Client ID for Authorized Application
+      # client_id=
+
+      ## Client Secret for Authorized Application
+      # client_secret=
+  ## [[[svn]]
+      ## Base URL to Remote Server
+      # remote_url=https://github.com/cloudera/hue/tree/master
+
+      ## Base URL to Version Control API
+      # api_url=https://api.github.com
+
+      # These will be necessary when you want to write back to the repository.
+      ## Client ID for Authorized Application
+      # client_id=
+
+      ## Client Secret for Authorized Application
+      # client_secret=
+
   # Configuration options for specifying the Desktop Database. For more info,
   # see http://docs.djangoproject.com/en/1.4/ref/settings/#database-engine
   # ------------------------------------------------------------------------
@@ -628,18 +666,6 @@
   ## Flag to enable the creation of a coordinator for the current SQL query.
   # enable_query_scheduling=false
 
-  ## Base URL to Remote GitHub Server
-  # github_remote_url=https://github.com
-
-  ## Base URL to GitHub API
-  # github_api_url=https://api.github.com
-
-  ## Client ID for Authorized GitHub Application
-  # github_client_id=
-
-  ## Client Secret for Authorized GitHub Application
-  # github_client_secret=
-
   # One entry for each type of snippet.
   [[interpreters]]
     # Define the name and how to connect and execute the language.

+ 38 - 12
desktop/conf/pseudo-distributed.ini.tmpl

@@ -511,6 +511,44 @@
           # The username attribute in the LDAP schema
           ## group_name_attr=cn
 
+  # Configuration options for specifying the Source Version Control.
+  # ----------------------------------------------------------------
+  [[vcs]]
+
+  ## [[[git-read-only]]]
+      ## Base URL to Remote Server
+      # remote_url=https://github.com/cloudera/hue/tree/master
+
+      ## Base URL to Version Control API
+      # api_url=https://api.github.com
+  ## [[[github]]]
+
+      ## Base URL to Remote Server
+      # remote_url=https://github.com/cloudera/hue/tree/master
+
+      ## Base URL to Version Control API
+      # api_url=https://api.github.com
+
+      # These will be necessary when you want to write back to the repository.
+      ## Client ID for Authorized Application
+      # client_id=
+
+      ## Client Secret for Authorized Application
+      # client_secret=
+  ## [[[svn]]
+      ## Base URL to Remote Server
+      # remote_url=https://github.com/cloudera/hue/tree/master
+
+      ## Base URL to Version Control API
+      # api_url=https://api.github.com
+
+      # These will be necessary when you want to write back to the repository.
+      ## Client ID for Authorized Application
+      # client_id=
+
+      ## Client Secret for Authorized Application
+      # client_secret=
+
   # Configuration options for specifying the Desktop Database. For more info,
   # see http://docs.djangoproject.com/en/1.4/ref/settings/#database-engine
   # ------------------------------------------------------------------------
@@ -630,18 +668,6 @@
   ## Flag to enable the creation of a coordinator for the current SQL query.
   # enable_query_scheduling=false
 
-  ## Base URL to Remote GitHub Server
-  # github_remote_url=https://github.com
-
-  ## Base URL to GitHub API
-  # github_api_url=https://api.github.com
-
-  ## Client ID for Authorized GitHub Application
-  # github_client_id=
-
-  ## Client Secret for Authorized GitHub Application
-  # github_client_secret=
-
   # One entry for each type of snippet.
   [[interpreters]]
     # Define the name and how to connect and execute the language.

+ 37 - 1
desktop/core/src/desktop/conf.py

@@ -29,7 +29,7 @@ from desktop.redaction.engine import parse_redaction_policy_from_file
 from desktop.lib.conf import Config, ConfigSection, UnspecifiedConfigSection,\
                              coerce_bool, coerce_csv, coerce_json_dict,\
                              validate_path, list_of_compiled_res, coerce_str_lowercase, \
-                             coerce_password_from_script
+                             coerce_password_from_script, coerce_string
 from desktop.lib.i18n import force_unicode
 from desktop.lib.paths import get_desktop_root
 
@@ -420,6 +420,42 @@ ALLOWED_HOSTS = Config(
   help=_('Comma separated list of strings representing the host/domain names that the Hue server can serve.')
 )
 
+VCS = UnspecifiedConfigSection(
+  "vcs",
+  help="One entry for each Version Control",
+  each=ConfigSection(
+    help="""Configuration options for source version control used to list and
+            save files from the editor. Example: Git, SVN""",
+    members=dict(
+      REMOTE_URL = Config(
+        key="remote_url",
+        help=_("Base URL to Interface Remote Server"),
+        default='https://github.com/cloudera/hue/',
+        type=coerce_string,
+      ),
+      API_URL = Config(
+        key="api_url",
+        help=_("Base URL to Interface API"),
+        default='https://api.github.com',
+        type=coerce_string,
+      ),
+      CLIENT_ID = Config(
+        key="client_id",
+        help=_("The Client ID of the Interface application."),
+        type=coerce_string,
+        default=""
+      ),
+      CLIENT_SECRET = Config(
+        key="client_secret",
+        help=_("The Client Secret of the Interface application."),
+        type=coerce_string,
+        default=""
+      )
+    )
+  )
+)
+
+
 def default_secure_cookie():
   """Enable secure cookies if HTTPS is enabled."""
   return is_https_enabled()

+ 16 - 0
desktop/core/src/desktop/lib/vcs/__init__.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 30 - 0
desktop/core/src/desktop/lib/vcs/api.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from desktop.api2 import api_error_handler
+from desktop.lib.vcs.apis.base_api import get_api
+
+
+@api_error_handler
+def authorize(request):
+  interface = request.GET.get('interface', 'git-read-only')
+  return get_api(interface).authorize(request)
+
+@api_error_handler
+def contents(request):
+  interface = request.GET.get('interface', 'git-read-only')
+  return get_api(interface).contents(request)

+ 16 - 0
desktop/core/src/desktop/lib/vcs/apis/__init__.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 50 - 0
desktop/core/src/desktop/lib/vcs/apis/base_api.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from django.utils.translation import ugettext as _
+
+from desktop.lib.exceptions_renderable import PopupException
+
+
+LOG = logging.getLogger(__name__)
+
+GITHUB_OFFICIAL = 'github'
+GIT_READ_ONLY = 'git-read-only'
+
+
+def get_api(interface):
+  from desktop.lib.vcs.apis.github_api import GithubApi
+  from desktop.lib.vcs.apis.github_readonly_api import GithubReadOnlyApi
+
+  if interface == GITHUB_OFFICIAL:
+    return GithubApi()
+  elif interface == GIT_READ_ONLY:
+    return GithubReadOnlyApi()
+  else:
+    raise PopupException(_('Interface %s is unknown') % interface)
+
+
+class Api(object):
+
+  def __init__(self):
+    pass
+
+  def authorize(self, request): return {}
+
+  def contents(self, request): return {}

+ 85 - 0
desktop/core/src/desktop/lib/vcs/apis/github_api.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+from django.http import HttpResponseBadRequest, HttpResponseRedirect
+from django.utils.translation import ugettext as _
+from django.views.decorators.http import require_GET
+
+from desktop.lib.django_util import JsonResponse
+
+from desktop.lib.vcs.github_client import GithubClient
+from desktop.lib.vcs.apis.base_api import Api
+
+
+class GithubApi(Api):
+
+  def __init__(self):
+    self.request = None
+
+  @require_GET
+  def contents(self, request):
+    response = {'status': -1}
+
+    api = GithubClient(access_token=request.session.get('github_access_token'))
+
+    response['url'] = url = request.GET.get('url')
+
+    if url:
+      owner, repo, branch, filepath = api.parse_github_url(url)
+      content = api.get_file_contents(owner, repo, filepath, branch)
+      try:
+        response['content'] = json.loads(content)
+      except ValueError:
+        # Content is not JSON-encoded so return plain-text
+        response['content'] = content
+      response['status'] = 0
+    else:
+      return HttpResponseBadRequest(_('url param is required'))
+
+    return JsonResponse(response)
+
+  def authorize(self, request):
+    access_token = request.session.get('github_access_token')
+    if access_token and GithubClient.is_authenticated(access_token):
+      response = {
+        'status': 0,
+        'message': _('User is already authenticated to GitHub.')
+      }
+      return JsonResponse(response)
+    else:
+      auth_url = GithubClient.get_authorization_url()
+      request.session['github_callback_redirect'] = request.GET.get('currentURL')
+      request.session['github_callback_fetch'] = request.GET.get('fetchURL')
+      response = {
+        'status': -1,
+        'auth_url':auth_url
+      }
+      if (request.is_ajax()):
+        return JsonResponse(response)
+
+      return HttpResponseRedirect(auth_url)
+
+  def callback(self, request):
+    redirect_base = request.session['github_callback_redirect'] + "&github_status="
+    if 'code' in request.GET:
+      session_code = request.GET.get('code')
+      request.session['github_access_token'] = GithubClient.get_access_token(session_code)
+      return HttpResponseRedirect(redirect_base + "0&github_fetch=" + request.session['github_callback_fetch'])
+    else:
+      return HttpResponseRedirect(redirect_base + "-1&github_fetch=" + request.session['github_callback_fetch'])

+ 116 - 0
desktop/core/src/desktop/lib/vcs/apis/github_readonly_api.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+import urllib
+import urlparse
+
+from django.http import HttpResponseBadRequest
+from django.utils.translation import ugettext as _
+
+from desktop.lib.django_util import JsonResponse
+from desktop.lib.rest.http_client import HttpClient, RestException
+from desktop.lib.rest import resource
+
+from desktop.conf import VCS
+from desktop.lib.vcs.apis.base_api import Api, GIT_READ_ONLY
+from desktop.lib.vcs.github_client import GithubClientException
+
+LOG = logging.getLogger(__name__)
+
+
+class GithubReadOnlyApi(Api):
+  """
+  https://developer.github.com/v3/
+  """
+
+  OWNER_RE = "(?P<owner>[A-Za-z0-9](?:-?[A-Za-z0-9]){0,38})"
+  REPO_RE = "(?P<repo>[\w\.@\:\-~]+)"
+  BRANCH_RE = "(?P<branch>[\w\.@\:\-~]+)"
+
+  DEFAULT_SCOPES = ['repo', 'user']
+
+  def __init__(self):
+    self._remote_url = VCS[GIT_READ_ONLY].REMOTE_URL.get().strip('/')
+    self._api_url = VCS[GIT_READ_ONLY].API_URL.get().strip('/')
+
+    self._client = HttpClient(self._api_url, logger=LOG)
+    self._root = resource.Resource(self._client)
+
+  def contents(self, request):
+    """
+    GET /repos/:owner/:repo/contents/:path
+    https://developer.github.com/v3/repos/contents/#get-contents
+    """
+    response = {'status': -1}
+    filepath = request.GET.get('path', '/')
+    filepath = self._clean_path(filepath)
+
+    if self._remote_url:
+      owner, repo, branch = self.parse_github_url(self._remote_url)
+      content = self._get_contents(owner, repo, filepath)
+      response['files'] = _massage_content(content)
+      response['status'] = 0
+    else:
+      return HttpResponseBadRequest(_('url param is required'))
+    return JsonResponse(response)
+
+  def authorize(self, request):
+    pass
+
+  def parse_github_url(self, url):
+    """
+    Given a base URL to a Github repository, return a tuple of the owner, repo, branch
+    :param url: base URL to repo (e.g. - https://github.com/cloudera/hue/tree/master)
+    :return: tuple of strings (e.g. - ('cloudera', 'hue', 'master'))
+    """
+    match = self.github_url_regex.search(url)
+    if match:
+      return match.group('owner'), match.group('repo'), match.group('branch')
+    else:
+      raise ValueError('GitHub URL is not formatted correctly: %s' % url)
+
+  @property
+  def github_url_regex(self):
+    return re.compile('%s/%s/%s/tree/%s' % (self._get_base_url(), self.OWNER_RE, self.REPO_RE, self.BRANCH_RE))
+
+  def _get_base_url(self):
+    split_url = urlparse.urlsplit(self._remote_url)
+    return urlparse.urlunsplit((split_url.scheme, split_url.netloc, '', "", ""))
+
+  def _clean_path(self, filepath):
+    cleaned_path = filepath.strip('/')
+    cleaned_path = urllib.unquote(cleaned_path)
+    return cleaned_path
+
+  def _get_contents(self, owner, repo, path):
+    try:
+      return self._root.get('repos/%s/%s/contents/%s' % (owner, repo, path))
+    except RestException, e:
+      raise GithubClientException('Could not find GitHub object, check owner, repo or path: %s' % e)
+
+
+def _massage_content(content):
+  response = []
+  for file in content:
+    file['stats'] = {
+      'size': file.get('size', 0),
+      'path': file.get('path', '')
+    }
+    response.append(file)
+  return response

+ 10 - 9
desktop/libs/notebook/src/notebook/github.py → desktop/core/src/desktop/lib/vcs/github_client.py

@@ -25,7 +25,8 @@ import urllib
 from desktop.lib.rest.http_client import HttpClient, RestException
 from desktop.lib.rest import resource
 
-from notebook.conf import GITHUB_REMOTE_URL, GITHUB_API_URL, GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET
+from desktop.conf import VCS
+from desktop.lib.vcs.apis.base_api import GITHUB_OFFICIAL
 
 
 LOG = logging.getLogger(__name__)
@@ -49,8 +50,8 @@ class GithubClient(object):
 
 
   def __init__(self, **options):
-    self._github_base_url = options.get('remote_url', GITHUB_REMOTE_URL.get()).strip('/')
-    self._api_url = options.get('api_url', GITHUB_API_URL.get()).strip('/')
+    self._github_base_url = options.get('remote_url', VCS[GITHUB_OFFICIAL].REMOTE_URL.get()).strip('/')
+    self._api_url = options.get('api_url', VCS[GITHUB_OFFICIAL].API_URL.get()).strip('/')
 
     self._client = HttpClient(self._api_url, logger=LOG)
     self._root = resource.Resource(self._client)
@@ -68,8 +69,8 @@ class GithubClient(object):
     """
     https://developer.github.com/guides/basics-of-authentication/
     """
-    remote_url = options.get('remote_url', GITHUB_REMOTE_URL.get()).strip('/')
-    client_id = options.get('client_id', GITHUB_CLIENT_ID.get())
+    remote_url = options.get('remote_url', VCS[GITHUB_OFFICIAL].REMOTE_URL.get()).strip('/')
+    client_id = options.get('client_id', VCS[GITHUB_OFFICIAL].CLIENT_ID.get())
     scopes_list = options.get('scopes_list', cls.DEFAULT_SCOPES)
     scopes = ','.join(scopes_list)
     return '%s/login/oauth/authorize?scope=%s&client_id=%s' % (remote_url, scopes, client_id)
@@ -77,9 +78,9 @@ class GithubClient(object):
 
   @classmethod
   def get_access_token(cls, session_code, **options):
-    remote_url = options.get('remote_url', GITHUB_REMOTE_URL.get()).strip('/')
-    client_id = options.get('client_id', GITHUB_CLIENT_ID.get())
-    client_secret = options.get('client_secret', GITHUB_CLIENT_SECRET.get())
+    remote_url = options.get('remote_url', VCS[GITHUB_OFFICIAL].REMOTE_URL.get()).strip('/')
+    client_id = options.get('client_id', VCS[GITHUB_OFFICIAL].CLIENT_ID.get())
+    client_secret = options.get('client_secret', VCS[GITHUB_OFFICIAL].CLIENT_SECRET.get())
 
     try:
       client = HttpClient(remote_url, logger=LOG)
@@ -104,7 +105,7 @@ class GithubClient(object):
 
   @classmethod
   def is_authenticated(cls, access_token, **options):
-    api_url = options.get('api_url', GITHUB_API_URL.get()).strip('/')
+    api_url = options.get('api_url', VCS[GITHUB_OFFICIAL].API_URL.get()).strip('/')
 
     try:
       client = HttpClient(api_url, logger=LOG)

+ 48 - 0
desktop/core/src/desktop/static/desktop/js/apiHelper.js

@@ -56,6 +56,7 @@ var ApiHelper = (function () {
   var DOCUMENTS_API = "/desktop/api2/doc/";
   var DOCUMENTS_SEARCH_API = "/desktop/api2/docs/";
   var HDFS_API_PREFIX = "/filebrowser/view=";
+  var GIT_API_PREFIX = "/desktop/api/vcs/contents/";
   var S3_API_PREFIX = "/filebrowser/view=S3A://";
   var IMPALA_INVALIDATE_API = '/impala/api/invalidate';
   var CONFIG_SAVE_API = '/desktop/api/configurations/save/';
@@ -102,6 +103,10 @@ var ApiHelper = (function () {
       $.totalStorage(self.getAssistCacheIdentifier({ sourceType: 'hdfs' }), {});
     });
 
+    huePubSub.subscribe('assist.clear.git.cache', function () {
+      $.totalStorage(self.getAssistCacheIdentifier({ sourceType: 'git' }), {});
+    });
+
     huePubSub.subscribe('assist.clear.s3.cache', function () {
       $.totalStorage(self.getAssistCacheIdentifier({ sourceType: 's3' }), {});
     });
@@ -124,6 +129,7 @@ var ApiHelper = (function () {
         clearAll: true
       });
       huePubSub.publish('assist.clear.hdfs.cache');
+      huePubSub.publish('assist.clear.git.cache');
       huePubSub.publish('assist.clear.s3.cache');
       huePubSub.publish('assist.clear.collections.cache');
       huePubSub.publish('assist.clear.hbase.cache');
@@ -375,6 +381,48 @@ var ApiHelper = (function () {
     }));
   };
 
+  /**
+   * @param {Object} options
+   * @param {Function} options.successCallback
+   * @param {Function} [options.errorCallback]
+   * @param {boolean} [options.silenceErrors]
+   * @param {Number} [options.timeout]
+   *
+   * @param {string[]} options.pathParts
+   */
+  ApiHelper.prototype.fetchGitPath = function (options) {
+    var self = this;
+    var url = GIT_API_PREFIX + '?path=' + options.pathParts.join("/");
+    var fetchFunction = function (storeInCache) {
+      if (options.timeout === 0) {
+        self.assistErrorCallback(options)({ status: -1 });
+        return;
+      }
+      $.ajax({
+        dataType: "json",
+        url: url,
+        timeout: options.timeout,
+        success: function (data) {
+          if (!data.error && !self.successResponseIsError(data) && typeof data.files !== 'undefined' && data.files !== null) {
+            if (data.files.length > 2) {
+              storeInCache(data);
+            }
+            options.successCallback(data);
+          } else {
+            self.assistErrorCallback(options)(data);
+          }
+        }
+      })
+      .fail(self.assistErrorCallback(options));
+    };
+
+    fetchCached.bind(self)($.extend({}, options, {
+      sourceType: 'git',
+      url: url,
+      fetchFunction: fetchFunction
+    }));
+  };
+
   /**
    * @param {Object} options
    * @param {Function} options.successCallback

+ 163 - 0
desktop/core/src/desktop/static/desktop/js/assist/assistGitEntry.js

@@ -0,0 +1,163 @@
+// Licensed to Cloudera, Inc. under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  Cloudera, Inc. licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+var AssistGitEntry = (function () {
+
+  /**
+   * @param {object} options
+   * @param {object} options.definition
+   * @param {string} options.definition.name
+   * @param {string} options.definition.type (file, dir)
+   * @param {AssistGitEntry} options.parent
+   * @param {ApiHelper} options.apiHelper
+   * @constructor
+   */
+  function AssistGitEntry (options) {
+    var self = this;
+
+    self.definition = options.definition;
+    self.apiHelper = options.apiHelper;
+    self.parent = options.parent;
+    self.path = '';
+    if (self.parent !== null) {
+      self.path = self.parent.path;
+      if (self.parent.path !== '/') {
+        self.path += '/'
+      }
+    }
+    self.path += self.definition.name;
+
+    self.entries = ko.observableArray([]);
+
+    self.loaded = false;
+    self.loading = ko.observable(false);
+    self.loadingMore = ko.observable(false);
+    self.hasErrors = ko.observable(false);
+    self.open = ko.observable(false);
+
+    self.open.subscribe(function(newValue) {
+      if (newValue && self.entries().length === 0) {
+        self.loadEntries();
+      }
+    });
+
+    self.hasEntries = ko.computed(function() {
+      return self.entries().length > 0;
+    });
+  }
+
+  AssistGitEntry.prototype.dblClick = function () {
+    var self = this;
+    huePubSub.publish('assist.dblClickGitItem', self);
+  };
+
+  AssistGitEntry.prototype.loadEntries = function(callback) {
+    var self = this;
+    if (self.loading()) {
+      return;
+    }
+    self.loading(true);
+    self.hasErrors(false);
+
+    var successCallback = function(data) {
+      var filteredFiles = $.grep(data.files, function (file) {
+        return file.name !== '.' && file.name !== '..';
+      });
+      self.entries($.map(filteredFiles, function (file) {
+        return new AssistGitEntry({
+          definition: file,
+          parent: self,
+          apiHelper: self.apiHelper
+        })
+      }));
+      self.loaded = true;
+      self.loading(false);
+      if (callback) {
+        callback();
+      }
+    };
+
+    var errorCallback = function () {
+      self.hasErrors(true);
+      self.loading(false);
+      if (callback) {
+        callback();
+      }
+    };
+
+    self.apiHelper.fetchGitPath({
+      pathParts: self.getHierarchy(),
+      successCallback: successCallback,
+      errorCallback: errorCallback
+    })
+  };
+
+  AssistGitEntry.prototype.loadDeep = function(folders, callback) {
+    var self = this;
+
+    if (folders.length === 0) {
+      callback(self);
+      return;
+    }
+
+    var findNextAndLoadDeep = function () {
+      var nextName = folders.shift();
+      var foundEntry = $.grep(self.entries(), function (entry) {
+        return entry.definition.name === nextName && entry.definition.type === 'dir';
+      });
+      if (foundEntry.length === 1) {
+        foundEntry[0].loadDeep(folders, callback);
+      } else if (! self.hasErrors()) {
+        callback(self);
+      }
+    };
+
+    if (! self.loaded) {
+      self.loadEntries(findNextAndLoadDeep);
+    } else {
+      findNextAndLoadDeep();
+    }
+  };
+
+  AssistGitEntry.prototype.getHierarchy = function () {
+    var self = this;
+    var parts = [];
+    var entry = self;
+    while (entry != null) {
+      parts.push(entry.definition.name);
+      entry = entry.parent;
+    }
+    parts.reverse();
+    return parts;
+  };
+
+  AssistGitEntry.prototype.toggleOpen = function () {
+    var self = this;
+    if (self.definition.type === 'file') {
+      return;
+    }
+    self.open(!self.open());
+    if (self.definition.name === '..') {
+      if (self.parent.parent) {
+        huePubSub.publish('assist.selectGitEntry', self.parent.parent);
+      }
+    } else {
+      huePubSub.publish('assist.selectGitEntry', self);
+    }
+  };
+
+  return AssistGitEntry;
+})();

+ 129 - 2
desktop/core/src/desktop/templates/assist.mako

@@ -18,7 +18,7 @@
 from django.utils.translation import ugettext as _
 
 from desktop import conf
-from desktop.conf import USE_NEW_SIDE_PANELS
+from desktop.conf import USE_NEW_SIDE_PANELS, VCS
 from desktop.lib.i18n import smart_unicode
 from desktop.views import _ko
 
@@ -31,6 +31,7 @@ from notebook.conf import ENABLE_QUERY_BUILDER
 <script src="${ static('desktop/js/assist/assistDbEntry.js') }"></script>
 <script src="${ static('desktop/js/assist/assistDbSource.js') }"></script>
 <script src="${ static('desktop/js/assist/assistHdfsEntry.js') }"></script>
+<script src="${ static('desktop/js/assist/assistGitEntry.js') }"></script>
 <script src="${ static('desktop/js/assist/assistS3Entry.js') }"></script>
 <script src="${ static('desktop/js/assist/assistCollectionEntry.js') }"></script>
 <script src="${ static('desktop/js/assist/assistHBaseEntry.js') }"></script>
@@ -308,6 +309,73 @@ from notebook.conf import ENABLE_QUERY_BUILDER
     </div>
   </script>
 
+
+  <script type="text/html" id="git-details-title">
+    <span data-bind="text: definition.name"></span>
+  </script>
+
+  <script type="text/html" id="assist-git-header-actions">
+    <div class="assist-db-header-actions" style="margin-top: -1px;">
+      <a class="inactive-action" href="javascript:void(0)" data-bind="click: function () { huePubSub.publish('assist.git.refresh'); }"><i class="pointer fa fa-refresh" data-bind="css: { 'fa-spin blue' : loading }" title="${_('Manual refresh')}"></i></a>
+    </div>
+  </script>
+
+  <script type="text/html" id="assist-git-inner-panel">
+    <div class="assist-inner-panel">
+      <div class="assist-flex-panel">
+        <!-- ko with: selectedGitEntry -->
+        <div class="assist-flex-header assist-breadcrumb" >
+          <!-- ko if: parent !== null -->
+          <a href="javascript: void(0);" data-bind="click: function () { huePubSub.publish('assist.selectGitEntry', parent); }">
+            <i class="fa fa-chevron-left" style="font-size: 15px;margin-right:8px;"></i>
+            <i class="fa fa-folder-o" style="font-size: 14px; line-height: 16px; vertical-align: top; margin-right:4px;"></i>
+            <span style="font-size: 14px;line-height: 16px;vertical-align: top;" data-bind="text: path"></span>
+          </a>
+          <!-- /ko -->
+          <!-- ko if: parent === null -->
+          <div style="padding-left: 5px;">
+            <i class="fa fa-folder-o" style="font-size: 14px; line-height: 16px;vertical-align: top; margin-right:4px;"></i>
+            <span style="font-size: 14px;line-height: 16px;vertical-align: top;" data-bind="text: path"></span>
+          </div>
+          <!-- /ko -->
+          <!-- ko template: 'assist-git-header-actions' --><!-- /ko -->
+        </div>
+        <div class="assist-flex-fill assist-git-scrollable">
+          <div data-bind="visible: ! loading() && ! hasErrors()" style="position: relative;">
+            <!-- ko hueSpinner: { spin: loadingMore, overlay: true } --><!-- /ko -->
+            <ul class="assist-tables" data-bind="foreachVisible: { data: entries, minHeight: 20, container: '.assist-git-scrollable' }">
+              <li class="assist-entry assist-table-link" style="position: relative;" data-bind="visibleOnHover: { 'selector': '.assist-actions' }">
+
+                <a href="javascript:void(0)" class="assist-entry assist-table-link" data-bind="multiClick: { click: toggleOpen, dblClick: dblClick }, attr: {'title': definition.name }">
+                  <!-- ko if: definition.type === 'dir' -->
+                  <i class="fa fa-fw fa-folder-o muted valign-middle"></i>
+                  <!-- /ko -->
+                  <!-- ko if: definition.type === 'file' -->
+                  <i class="fa fa-fw fa-file-o muted valign-middle"></i>
+                  <!-- /ko -->
+                  <span draggable="true" data-bind="text: definition.name, draggableText: { text: '\'' + path + '\'', meta: {'type': 'git', 'definition': definition} }"></span>
+                </a>
+              </li>
+            </ul>
+            <!-- ko if: !loading() && entries().length === 0 -->
+            <ul class="assist-tables">
+              <li class="assist-entry" style="font-style: italic;">${_('Empty directory')}</li>
+            </ul>
+            <!-- /ko -->
+          </div>
+          <!-- ko hueSpinner: { spin: loading, center: true, size: 'large' } --><!-- /ko -->
+          <div class="assist-errors" data-bind="visible: ! loading() && hasErrors()">
+            <span>${ _('Error loading contents.') }</span>
+          </div>
+        </div>
+        <!-- /ko -->
+      </div>
+      <!-- ko with: $parents[1] -->
+      <!-- ko template: { if: searchActive() && searchInput() !== '' && navigatorEnabled(), name: 'nav-search-result' } --><!-- /ko -->
+      <!-- /ko -->
+    </div>
+  </script>
+
   <script type="text/html" id="hdfs-details-content">
     <!-- ko with: definition -->
     <div class="assist-details-wrap">
@@ -902,7 +970,7 @@ from notebook.conf import ENABLE_QUERY_BUILDER
        * @param {boolean} [options.rightAlignIcon] - Default false
        * @param {boolean} options.visible
        * @param {boolean} [options.showNavSearch] - Default true
-       * @param {(AssistDbPanel|AssistHdfsPanel|AssistDocumentsPanel|AssistS3Panel|AssistCollectionsPanel)} panelData
+       * @param {(AssistDbPanel|AssistHdfsPanel|AssistGitPanel|AssistDocumentsPanel|AssistS3Panel|AssistCollectionsPanel)} panelData
        * @constructor
        */
       function AssistInnerPanel (options) {
@@ -1157,6 +1225,51 @@ from notebook.conf import ENABLE_QUERY_BUILDER
         this.reload();
       };
 
+      /**
+       * @param {Object} options
+       * @param {ApiHelper} options.apiHelper
+       * @constructor
+       **/
+      function AssistGitPanel (options) {
+        var self = this;
+        self.apiHelper = ApiHelper.getInstance();
+
+        self.selectedGitEntry = ko.observable();
+        self.reload = function () {
+          var lastKnownPath = self.apiHelper.getFromTotalStorage('assist', 'currentGitPath', '${ home_dir }');
+          var parts = lastKnownPath.split('/');
+          parts.shift();
+
+          var currentEntry = new AssistGitEntry({
+            definition: {
+              name: '/',
+              type: 'dir'
+            },
+            parent: null,
+            apiHelper: self.apiHelper
+          });
+
+          currentEntry.loadDeep(parts, function (entry) {
+            self.selectedGitEntry(entry);
+            entry.open(true);
+          });
+        };
+
+        huePubSub.subscribe('assist.selectGitEntry', function (entry) {
+          self.selectedGitEntry(entry);
+          self.apiHelper.setInTotalStorage('assist', 'currentGitPath', entry.path);
+        });
+
+        huePubSub.subscribe('assist.git.refresh', function () {
+          huePubSub.publish('assist.clear.git.cache');
+          self.reload();
+        });
+      }
+
+      AssistGitPanel.prototype.init = function () {
+        this.reload();
+      };
+
 
       /**
        * @param {Object} options
@@ -1450,6 +1563,20 @@ from notebook.conf import ENABLE_QUERY_BUILDER
             rightAlignIcon: true,
             visible: params.visibleAssistPanels && params.visibleAssistPanels.indexOf('documents') !== -1
           }));
+
+          if (${ len(VCS.keys()) } > 0) {
+            self.availablePanels.push(new AssistInnerPanel({
+              panelData: new AssistGitPanel({
+                apiHelper: self.apiHelper
+              }),
+              apiHelper: self.apiHelper,
+              name: '${ _("GIT") }',
+              type: 'git',
+              icon: 'fa-github',
+              minHeight: 50,
+              rightAlignIcon: true
+            }));
+          }
         }
 
         self.performSearch = function () {

+ 5 - 0
desktop/core/src/desktop/urls.py

@@ -152,6 +152,11 @@ dynamic_patterns += patterns('useradmin.views',
   (r'^desktop/api/users/autocomplete', 'list_for_autocomplete'),
 )
 
+dynamic_patterns += patterns('desktop.lib.vcs.api',
+  (r'^desktop/api/vcs/contents/?$', 'contents'),
+  (r'^desktop/api/vcs/authorize/?$', 'authorize'),
+)
+
 # Metrics specific
 if METRICS.ENABLE_WEB_METRICS.get():
   dynamic_patterns += patterns('',

+ 0 - 60
desktop/libs/notebook/src/notebook/api.py

@@ -22,7 +22,6 @@ import sqlparse
 
 from django.core.urlresolvers import reverse
 from django.db.models import Q
-from django.http import HttpResponseBadRequest, HttpResponseRedirect
 from django.utils.translation import ugettext as _
 from django.views.decorators.http import require_GET, require_POST
 
@@ -32,7 +31,6 @@ from desktop.models import Document2, Document
 
 from notebook.connectors.base import get_api, Notebook, QueryExpired, SessionExpired, QueryError
 from notebook.decorators import api_error_handler, check_document_access_permission, check_document_modify_permission
-from notebook.github import GithubClient
 from notebook.models import escape_rows
 from notebook.views import upgrade_session_properties
 
@@ -608,64 +606,6 @@ def format(request):
   return JsonResponse(response)
 
 
-@require_GET
-@api_error_handler
-def github_fetch(request):
-  response = {'status': -1}
-
-  api = GithubClient(access_token=request.session.get('github_access_token'))
-
-  response['url'] = url = request.GET.get('url')
-
-  if url:
-    owner, repo, branch, filepath = api.parse_github_url(url)
-    content = api.get_file_contents(owner, repo, filepath, branch)
-    try:
-      response['content'] = json.loads(content)
-    except ValueError:
-      # Content is not JSON-encoded so return plain-text
-      response['content'] = content
-    response['status'] = 0
-  else:
-    return HttpResponseBadRequest(_('url param is required'))
-
-  return JsonResponse(response)
-
-
-@api_error_handler
-def github_authorize(request):
-  access_token = request.session.get('github_access_token')
-  if access_token and GithubClient.is_authenticated(access_token):
-    response = {
-      'status': 0,
-      'message': _('User is already authenticated to GitHub.')
-    }
-    return JsonResponse(response)
-  else:
-    auth_url = GithubClient.get_authorization_url()
-    request.session['github_callback_redirect'] = request.GET.get('currentURL')
-    request.session['github_callback_fetch'] = request.GET.get('fetchURL')
-    response = {
-      'status': -1,
-      'auth_url':auth_url
-    }
-    if (request.is_ajax()):
-      return JsonResponse(response)
-
-    return HttpResponseRedirect(auth_url)
-
-
-@api_error_handler
-def github_callback(request):
-  redirect_base = request.session['github_callback_redirect'] + "&github_status="
-  if 'code' in request.GET:
-    session_code = request.GET.get('code')
-    request.session['github_access_token'] = GithubClient.get_access_token(session_code)
-    return HttpResponseRedirect(redirect_base + "0&github_fetch=" + request.session['github_callback_fetch'])
-  else:
-    return HttpResponseRedirect(redirect_base + "-1&github_fetch=" + request.session['github_callback_fetch'])
-
-
 @require_POST
 @check_document_access_permission()
 @api_error_handler

+ 0 - 29
desktop/libs/notebook/src/notebook/conf.py

@@ -139,35 +139,6 @@ ENABLE_BATCH_EXECUTE = Config(
 )
 
 
-GITHUB_REMOTE_URL = Config(
-    key="github_remote_url",
-    help=_t("Base URL to GitHub Remote Server"),
-    default='https://github.com',
-    type=coerce_string,
-)
-
-GITHUB_API_URL = Config(
-    key="github_api_url",
-    help=_t("Base URL to GitHub API"),
-    default='https://api.github.com',
-    type=coerce_string,
-)
-
-GITHUB_CLIENT_ID = Config(
-    key="github_client_id",
-    help=_t("The Client ID of the GitHub application."),
-    type=coerce_string,
-    default=""
-)
-
-GITHUB_CLIENT_SECRET = Config(
-    key="github_client_secret",
-    help=_t("The Client Secret of the GitHub application."),
-    type=coerce_string,
-    default=""
-)
-
-
 def _default_interpreters():
   INTERPRETERS.set_for_testing(OrderedDict((
       ('hive', {

+ 0 - 7
desktop/libs/notebook/src/notebook/urls.py

@@ -96,11 +96,4 @@ urlpatterns += patterns('notebook.api',
   url(r'^api/autocomplete/(?P<server>\w+)/(?P<database>[\w._\-0-9]+)/(?P<table>\w+)/?$', 'autocomplete', name='api_autocomplete_columns'),
   url(r'^api/sample/(?P<server>\w+)/(?P<database>[\w._\-0-9]+)/(?P<table>\w+)/?$', 'get_sample_data', name='api_sample_data'),
   url(r'^api/sample/(?P<server>\w+)/(?P<database>[\w._\-0-9]+)/(?P<table>\w+)/(?P<column>\w+)/?$', 'get_sample_data', name='api_sample_data_column'),
-)
-
-# Github
-urlpatterns += patterns('notebook.api',
-  url(r'^api/github/fetch/?$', 'github_fetch', name='github_fetch'),
-  url(r'^api/github/authorize/?$', 'github_authorize', name='github_authorize'),
-  url(r'^api/github/callback/?$', 'github_callback', name='github_callback'),
 )