Преглед на файлове

[logs] Refactor old Hue logs views into new public APIs (#3899)

- Current log methods in Hue are super old and returning server-side mako based view responses.
- This refactoring helps in improving the core logic and present it as a public API.

- This will also help the ongoing revamp of Hue server logs page to consume the new APIs. Once the new server logs page is ready, we can cleanup all related old code.
Harsh Gupta преди 11 месеца
родител
ревизия
979d867b34

+ 0 - 1
apps/filebrowser/src/filebrowser/api_test.py

@@ -67,7 +67,6 @@ class TestSimpleFileUploadAPI:
           ]
           try:
             response = upload_file(request)
-            print(response.content)
             response_data = json.loads(response.content)
 
             assert response.status_code == 200

+ 13 - 0
desktop/core/src/desktop/api_public.py

@@ -27,6 +27,7 @@ from desktop import api2 as desktop_api
 from desktop.auth.backend import rewrite_user
 from desktop.lib import fsmanager
 from desktop.lib.connectors import api as connector_api
+from desktop.log import api as logs_api
 from filebrowser import api as filebrowser_api, views as filebrowser_views
 from indexer import api3 as indexer_api3
 from metadata import optimizer_api
@@ -58,6 +59,18 @@ def get_banners(request):
   return desktop_api.get_banners(request)
 
 
+@api_view(["GET"])
+def get_hue_logs(request):
+  django_request = get_django_request(request)
+  return logs_api.get_hue_logs(django_request)
+
+
+@api_view(["GET"])
+def download_hue_logs(request):
+  django_request = get_django_request(request)
+  return logs_api.download_hue_logs(django_request)
+
+
 # Editor
 
 

+ 1 - 1
desktop/core/src/desktop/api_public_tests.py

@@ -43,7 +43,7 @@ class TestCoreApi():
         done = CUSTOM.BANNER_TOP_HTML.set_for_testing(configured_banner)
         get_banner_message.return_value = system_banner
 
-        response = self.unauthorized_client.get(reverse('api:core_banners'))
+        response = self.unauthorized_client.get(reverse('api:core_get_banners'))
 
         get_banner_message.assert_called()
         json_resp = json.loads(response.content)

+ 7 - 5
desktop/core/src/desktop/api_public_urls_v1.py

@@ -30,7 +30,9 @@ urlpatterns = [
 # Compatibility with "old" private API.
 # e.g. https://demo.gethue.com/notebook/api/execute/hive
 urlpatterns += [
-  re_path(r'^banners/?$', api_public.get_banners, name='core_banners'),
+  re_path(r'^banners/?$', api_public.get_banners, name='core_get_banners'),
+  re_path(r'^logs/?$', api_public.get_hue_logs, name='core_get_hue_logs'),
+  re_path(r'^logs/download/?$', api_public.download_hue_logs, name='core_download_hue_logs'),
   re_path(r'^get_config/?$', api_public.get_config),
   re_path(r'^get_namespaces/(?P<interface>[\w\-]+)/?$', api_public.get_context_namespaces),  # To remove
 ]
@@ -53,7 +55,7 @@ urlpatterns += [
   re_path(
     r'^editor/describe/(?P<database>[^/?]*)/(?P<table>[^/?]+)/stats(?:/(?P<column>[^/?]+))?/?$',
     api_public.describe,
-    name='editor_describe_column'
+    name='editor_describe_column',
   ),
   re_path(r'^editor/autocomplete/?$', api_public.autocomplete, name='editor_autocomplete_databases'),
   re_path(
@@ -83,9 +85,9 @@ urlpatterns += [
     name='editor_sample_data_column',
   ),
   re_path(
-      r"^editor/sample/(?P<database>[^/?]*)/(?P<table>[^/?]+)/(?P<column>[^/?]+)/(?P<nested>.+)/?$",
-      api_public.get_sample_data,
-      name="editor_sample_data_nested",
+    r"^editor/sample/(?P<database>[^/?]*)/(?P<table>[^/?]+)/(?P<column>[^/?]+)/(?P<nested>.+)/?$",
+    api_public.get_sample_data,
+    name="editor_sample_data_nested",
   ),
 ]
 

+ 214 - 0
desktop/core/src/desktop/log/api.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+import socket
+import logging
+import zipfile
+import tempfile
+from wsgiref.util import FileWrapper
+
+from django.http import HttpResponse
+
+from desktop.auth.backend import is_admin
+from desktop.lib.django_util import JsonResponse
+from desktop.lib.i18n import smart_str
+from desktop.log import DEFAULT_LOG_DIR
+
+LOG = logging.getLogger()
+
+
+# TODO: Improve error response further with better context -- Error UX Phase 2
+def api_error_handler(view_fn):
+  """
+  Decorator to handle exceptions and return a JSON response with an error message.
+  """
+
+  def decorator(*args, **kwargs):
+    try:
+      return view_fn(*args, **kwargs)
+    except Exception as e:
+      LOG.exception(f'Error running {view_fn.__name__}: {str(e)}')
+      return JsonResponse({'error': str(e)}, status=500)
+
+  return decorator
+
+
+@api_error_handler
+def get_hue_logs(request):
+  """
+  Retrieves the last X characters of log messages from the log file.
+
+  Args:
+    request: The HTTP request object.
+
+  Returns:
+    A JSON response containing the current hostname and log messages.
+
+  Raises:
+    403: If user accessing the endpoint in not a Hue admin.
+    404: If the log directory or file does not exist.
+    500: If an internal server error occurs.
+
+  Notes:
+    This endpoint retrieves the last X characters of log messages from the log file.
+    The log file is read up to the buffer size, and if it's smaller than the buffer size,
+    the previous log file is read to complete the buffer.
+  """
+  if not is_admin(request.user):
+    return HttpResponse("You must be a Hue admin to access this endpoint.", status=403)
+
+  # Buffer size for reading log files
+  LOG_BUFFER_SIZE = 32 * 1024
+
+  log_directory = os.getenv("DESKTOP_LOG_DIR", DEFAULT_LOG_DIR)
+  if not log_directory:
+    return HttpResponse('The log directory is not set or does not exist.', status=404)
+
+  log_file = os.path.join(log_directory, 'rungunicornserver.log')
+  if not os.path.exists(log_file):
+    return HttpResponse('The log file does not exist.', status=404)
+
+  log_file_size = os.path.getsize(log_file)
+
+  # Read the log file contents
+  buffer = _read_log_file(LOG_BUFFER_SIZE, log_file, log_file_size)
+
+  # If the log file is smaller than the buffer size, read the previous log file
+  if log_file_size < LOG_BUFFER_SIZE:
+    previous_log_file = os.path.join(log_directory, 'rungunicornserver.log.1')
+
+    if os.path.exists(previous_log_file):
+      prev_log_file_size = os.path.getsize(previous_log_file)
+      # Read the previous log file contents
+      buffer = _read_previous_log_file(LOG_BUFFER_SIZE, previous_log_file, prev_log_file_size, log_file_size) + buffer
+
+  response = {'hue_hostname': socket.gethostname(), 'logs': ''.join(buffer)}
+  return JsonResponse(response)
+
+
+@api_error_handler
+def download_hue_logs(request):
+  """
+  Downloads the last X characters of log messages from the log file as a zip attachment.
+
+  Args:
+    request: The HTTP request object.
+
+  Returns:
+    A zip file containing the log messages.
+
+  Raises:
+    403: If user accessing the endpoint in not a Hue admin.
+    404: If the log directory or log file does not exist.
+    500: If an internal server error occurs.
+
+  Notes:
+    This endpoint downloads the last X characters of log messages from the log file as a zip attachment.
+    The log file is read up to the buffer size, and if it's smaller than the buffer size,
+    the previous log file is read to complete the buffer.
+  """
+  if not is_admin(request.user):
+    return HttpResponse("You must be a Hue admin to access this endpoint.", status=403)
+
+  # Buffer size for reading log files for download (1 MiB)
+  LOG_DOWNLOAD_BUFFER_SIZE = 1024 * 1024
+
+  log_directory = os.getenv("DESKTOP_LOG_DIR", DEFAULT_LOG_DIR)
+  if not log_directory:
+    return HttpResponse('The log directory is not set or does not exist.', status=404)
+
+  log_file = os.path.join(log_directory, 'rungunicornserver.log')
+  if not os.path.exists(log_file):
+    return HttpResponse('The log file does not exist.', status=404)
+
+  log_file_size = os.path.getsize(log_file)
+
+  # Read the log file contents
+  buffer = _read_log_file(LOG_DOWNLOAD_BUFFER_SIZE, log_file, log_file_size)
+
+  # If the log file is smaller than the buffer size, read the previous log file
+  if log_file_size < LOG_DOWNLOAD_BUFFER_SIZE:
+    previous_log_file = os.path.join(log_directory, 'rungunicornserver.log.1')
+
+    if os.path.exists(previous_log_file):
+      prev_log_file_size = os.path.getsize(previous_log_file)
+      # Read the previous log file contents
+      buffer = _read_previous_log_file(LOG_DOWNLOAD_BUFFER_SIZE, previous_log_file, prev_log_file_size, log_file_size) + buffer
+
+  # Avoid loading large logs into memory by writing to a temp file line-by-line, allowing zipfile to handle the data more efficiently
+  log_tmp = tempfile.NamedTemporaryFile("w+t", encoding='utf-8')
+  for line in buffer:
+    log_tmp.write(smart_str(line, errors='replace'))
+  # Without flush, there is a chance to get truncated logs
+  log_tmp.flush()
+
+  tmp = tempfile.NamedTemporaryFile()
+  zip_file = zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED)
+  zip_file.write(log_tmp.name, f"hue-logs/hue-{time.time()}.log")
+  zip_file.close()
+  length = tmp.tell()
+
+  # If we don't seek to start of file, no bytes will be written
+  tmp.seek(0)
+  wrapper = FileWrapper(tmp)
+  # Return the zip file as a response and set the Content-Disposition header to force the browser to download the file
+  response = HttpResponse(wrapper, content_type="application/zip")
+  response['Content-Disposition'] = f'attachment; filename=hue-logs-{time.time()}.zip'
+  response['Content-Length'] = length
+
+  return response
+
+
+def _read_log_file(log_buffer_size, log_file, log_file_size):
+  """
+  Reads the log file contents up to the buffer size.
+
+  Args:
+    log_buffer_size: Maximum buffer size for reading logs.
+    log_file: The log file path.
+    log_file_size: The log file size.
+
+  Returns:
+    A list of log lines.
+  """
+  with open(log_file, 'rb') as fh:
+    if log_file_size > log_buffer_size:
+      fh.seek(log_file_size - log_buffer_size)
+    else:
+      fh.seek(0)
+    return [line.decode('utf-8') for line in fh.readlines()]
+
+
+def _read_previous_log_file(log_buffer_size, previous_log_file, prev_log_file_size, log_file_size):
+  """
+  Reads the previous log file contents up to the buffer size.
+
+  Args:
+    log_buffer_size: Maximum buffer size for reading logs.
+    previous_log_file: The previous log file path.
+    prev_log_file_size: The previous log file size.
+    log_file_size: The current log file size.
+
+  Returns:
+    A list of log lines.
+  """
+  with open(previous_log_file, 'rb') as fh1:
+    start_pos = max(0, prev_log_file_size - log_buffer_size - log_file_size)
+    fh1.seek(start_pos)
+    return [line.decode('utf-8') for line in fh1.readlines()]

+ 85 - 0
desktop/core/src/desktop/log/api_test.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import socket
+from unittest.mock import Mock, patch
+
+import pytest
+
+from desktop.lib.django_test_util import make_logged_in_client
+from desktop.log.api import get_hue_logs
+from useradmin.models import User
+
+
+@pytest.mark.django_db
+class TestHueLogs:
+  def setup_method(self):
+    self.client_admin = make_logged_in_client(username="test_admin", groupname="default", recreate=True, is_superuser=True)
+    self.user_admin = User.objects.get(username="test_admin")
+
+    self.client_not_admin = make_logged_in_client(username="test_not_admin", groupname="default", recreate=True, is_superuser=False)
+    self.user_not_admin = User.objects.get(username="test_not_admin")
+
+  def test_get_hue_logs_unauthorized(self):
+    request = Mock(method='GET', user=self.user_not_admin)
+
+    response = get_hue_logs(request)
+    res_content = response.content.decode('utf-8')
+
+    assert response.status_code == 403
+    assert res_content == 'You must be a Hue admin to access this endpoint.'
+
+  def test_log_directory_not_set(self):
+    with patch('desktop.log.api.os.getenv') as os_getenv:
+      request = Mock(method='GET', user=self.user_admin)
+      os_getenv.return_value = None
+
+      response = get_hue_logs(request)
+      res_content = response.content.decode('utf-8')
+
+      assert response.status_code == 404
+      assert res_content == 'The log directory is not set or does not exist.'
+
+  def test_log_file_not_found(self):
+    with patch('desktop.log.api.os.getenv') as os_getenv:
+      with patch('desktop.log.api.os.path.exists') as os_path_exist:
+        request = Mock(method='GET', user=self.user_admin)
+        os_getenv.return_value = '/var/log/hue/'
+        os_path_exist.return_value = False
+
+        response = get_hue_logs(request)
+        res_content = response.content.decode('utf-8')
+
+        assert response.status_code == 404
+        assert res_content == 'The log file does not exist.'
+
+  def test_get_hue_logs_success(self):
+    with patch('desktop.log.api.os') as mock_os:
+      with patch('desktop.log.api._read_log_file') as _read_log_file:
+        request = Mock(method='GET', user=self.user_admin)
+        _read_log_file.return_value = 'test log content'
+
+        mock_os.os_getenv.return_value = '/var/log/hue/'
+        mock_os.path.exists.return_value = True
+        mock_os.path.getsize.return_value = 32 * 1024 * 2  # Greater than log buffer size
+
+        response = get_hue_logs(request)
+        response_data = json.loads(response.content)
+
+        assert response.status_code == 200
+        assert response_data == {'hue_hostname': socket.gethostname(), 'logs': 'test log content'}