浏览代码

HUE-1744 [core] upgrade packaged thrift

Thrift 0.9.1 inclusion.
Thrift 0.9.1 uses TCP/IP v6 by default.
Updated thrift_util_test.py to use TCP/IP v6.
Abraham Elmahrek 12 年之前
父节点
当前提交
b14ee7569c
共有 39 个文件被更改,包括 3526 次插入448 次删除
  1. 2 2
      desktop/core/ext-py/thrift-0.9.1/Makefile.am
  2. 494 0
      desktop/core/ext-py/thrift-0.9.1/Makefile.in
  3. 0 0
      desktop/core/ext-py/thrift-0.9.1/README
  4. 247 0
      desktop/core/ext-py/thrift-0.9.1/compat/win32/stdint.h
  5. 2 0
      desktop/core/ext-py/thrift-0.9.1/setup.cfg
  6. 104 0
      desktop/core/ext-py/thrift-0.9.1/setup.py
  7. 5 3
      desktop/core/ext-py/thrift-0.9.1/src/TSCons.py
  8. 16 22
      desktop/core/ext-py/thrift-0.9.1/src/TSerialization.py
  9. 153 0
      desktop/core/ext-py/thrift-0.9.1/src/TTornado.py
  10. 60 13
      desktop/core/ext-py/thrift-0.9.1/src/Thrift.py
  11. 0 0
      desktop/core/ext-py/thrift-0.9.1/src/__init__.py
  12. 81 0
      desktop/core/ext-py/thrift-0.9.1/src/protocol/TBase.py
  13. 7 6
      desktop/core/ext-py/thrift-0.9.1/src/protocol/TBinaryProtocol.py
  14. 403 0
      desktop/core/ext-py/thrift-0.9.1/src/protocol/TCompactProtocol.py
  15. 550 0
      desktop/core/ext-py/thrift-0.9.1/src/protocol/TJSONProtocol.py
  16. 406 0
      desktop/core/ext-py/thrift-0.9.1/src/protocol/TProtocol.py
  17. 1 1
      desktop/core/ext-py/thrift-0.9.1/src/protocol/__init__.py
  18. 21 5
      desktop/core/ext-py/thrift-0.9.1/src/protocol/fastbinary.c
  19. 37 13
      desktop/core/ext-py/thrift-0.9.1/src/server/THttpServer.py
  20. 80 43
      desktop/core/ext-py/thrift-0.9.1/src/server/TNonblockingServer.py
  21. 118 0
      desktop/core/ext-py/thrift-0.9.1/src/server/TProcessPoolServer.py
  22. 22 23
      desktop/core/ext-py/thrift-0.9.1/src/server/TServer.py
  23. 0 0
      desktop/core/ext-py/thrift-0.9.1/src/server/__init__.py
  24. 59 10
      desktop/core/ext-py/thrift-0.9.1/src/transport/THttpClient.py
  25. 214 0
      desktop/core/ext-py/thrift-0.9.1/src/transport/TSSLSocket.py
  26. 44 17
      desktop/core/ext-py/thrift-0.9.1/src/transport/TSocket.py
  27. 21 17
      desktop/core/ext-py/thrift-0.9.1/src/transport/TTransport.py
  28. 43 3
      desktop/core/ext-py/thrift-0.9.1/src/transport/TTwisted.py
  29. 248 0
      desktop/core/ext-py/thrift-0.9.1/src/transport/TZlibTransport.py
  30. 1 1
      desktop/core/ext-py/thrift-0.9.1/src/transport/__init__.py
  31. 0 205
      desktop/core/ext-py/thrift/src/protocol/TProtocol.py
  32. 6 3
      desktop/core/src/desktop/lib/gen-py/djangothrift_test_gen/TestService-remote
  33. 14 10
      desktop/core/src/desktop/lib/gen-py/djangothrift_test_gen/TestService.py
  34. 4 2
      desktop/core/src/desktop/lib/gen-py/djangothrift_test_gen/constants.py
  35. 32 27
      desktop/core/src/desktop/lib/gen-py/djangothrift_test_gen/ttypes.py
  36. 15 0
      desktop/core/src/desktop/lib/python_util.py
  37. 8 6
      desktop/core/src/desktop/lib/thrift_util_test.py
  38. 5 15
      desktop/libs/hadoop/src/hadoop/mini_cluster.py
  39. 3 1
      desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

+ 2 - 2
desktop/core/ext-py/thrift/Makefile.am → desktop/core/ext-py/thrift-0.9.1/Makefile.am

@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-
+.NOTPARALLEL:
 DESTDIR ?= /
-EXTRA_DIST = setup.py src
+EXTRA_DIST = setup.py setup.cfg src compat
 
 all-local:
 	$(PYTHON) setup.py build

+ 494 - 0
desktop/core/ext-py/thrift-0.9.1/Makefile.in

@@ -0,0 +1,494 @@
+# Makefile.in generated by automake 1.11.3 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
+# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software
+# Foundation, Inc.
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+VPATH = @srcdir@
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+subdir = lib/py
+DIST_COMMON = README $(srcdir)/Makefile.am $(srcdir)/Makefile.in
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/aclocal/ax_boost_base.m4 \
+	$(top_srcdir)/aclocal/ax_check_openssl.m4 \
+	$(top_srcdir)/aclocal/ax_dmd.m4 \
+	$(top_srcdir)/aclocal/ax_javac_and_java.m4 \
+	$(top_srcdir)/aclocal/ax_lib_event.m4 \
+	$(top_srcdir)/aclocal/ax_lib_zlib.m4 \
+	$(top_srcdir)/aclocal/ax_prog_perl_modules.m4 \
+	$(top_srcdir)/aclocal/ax_signed_right_shift.m4 \
+	$(top_srcdir)/aclocal/ax_thrift_internal.m4 \
+	$(top_srcdir)/aclocal/libtool.m4 \
+	$(top_srcdir)/aclocal/ltoptions.m4 \
+	$(top_srcdir)/aclocal/ltsugar.m4 \
+	$(top_srcdir)/aclocal/ltversion.m4 \
+	$(top_srcdir)/aclocal/lt~obsolete.m4 \
+	$(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h \
+	$(top_builddir)/lib/cpp/src/thrift/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+SOURCES =
+DIST_SOURCES =
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ALLOCA = @ALLOCA@
+AMTAR = @AMTAR@
+ANT = @ANT@
+ANT_FLAGS = @ANT_FLAGS@
+AR = @AR@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BOOST_CPPFLAGS = @BOOST_CPPFLAGS@
+BOOST_LDFLAGS = @BOOST_LDFLAGS@
+BOOST_ROOT_PATH = @BOOST_ROOT_PATH@
+BUNDLER = @BUNDLER@
+CABAL = @CABAL@
+CABAL_CONFIGURE_FLAGS = @CABAL_CONFIGURE_FLAGS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CFLAGS = @CFLAGS@
+CLASSPATH = @CLASSPATH@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DLLTOOL = @DLLTOOL@
+DMD = @DMD@
+DMD_LIBEVENT_FLAGS = @DMD_LIBEVENT_FLAGS@
+DMD_OF_DIRSEP = @DMD_OF_DIRSEP@
+DMD_OPENSSL_FLAGS = @DMD_OPENSSL_FLAGS@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+D_EVENT_LIB_NAME = @D_EVENT_LIB_NAME@
+D_IMPORT_PREFIX = @D_IMPORT_PREFIX@
+D_LIB_NAME = @D_LIB_NAME@
+D_SSL_LIB_NAME = @D_SSL_LIB_NAME@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+ENABLE_COVERAGE = @ENABLE_COVERAGE@
+ERL = @ERL@
+ERLANG_INSTALL_LIB_DIR = @ERLANG_INSTALL_LIB_DIR@
+ERLANG_INSTALL_LIB_DIR_thrift = @ERLANG_INSTALL_LIB_DIR_thrift@
+ERLANG_LIB_DIR = @ERLANG_LIB_DIR@
+ERLC = @ERLC@
+ERLCFLAGS = @ERLCFLAGS@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+GCOV_CFLAGS = @GCOV_CFLAGS@
+GCOV_CXXFLAGS = @GCOV_CXXFLAGS@
+GCOV_LDFLAGS = @GCOV_LDFLAGS@
+GLIB_CFLAGS = @GLIB_CFLAGS@
+GLIB_LIBS = @GLIB_LIBS@
+GO = @GO@
+GOBJECT_CFLAGS = @GOBJECT_CFLAGS@
+GOBJECT_LIBS = @GOBJECT_LIBS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALLDIRS = @INSTALLDIRS@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+JAVA_PREFIX = @JAVA_PREFIX@
+LD = @LD@
+LDFLAGS = @LDFLAGS@
+LEX = @LEX@
+LEXLIB = @LEXLIB@
+LEX_OUTPUT_ROOT = @LEX_OUTPUT_ROOT@
+LIBEVENT_CPPFLAGS = @LIBEVENT_CPPFLAGS@
+LIBEVENT_LDFLAGS = @LIBEVENT_LDFLAGS@
+LIBEVENT_LIBS = @LIBEVENT_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBTOOL = @LIBTOOL@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBOBJS = @LTLIBOBJS@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MONO_CFLAGS = @MONO_CFLAGS@
+MONO_LIBS = @MONO_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OPENSSL_INCLUDES = @OPENSSL_INCLUDES@
+OPENSSL_LDFLAGS = @OPENSSL_LDFLAGS@
+OPENSSL_LIBS = @OPENSSL_LIBS@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PERL = @PERL@
+PERL_PREFIX = @PERL_PREFIX@
+PHP = @PHP@
+PHPUNIT = @PHPUNIT@
+PHP_CONFIG = @PHP_CONFIG@
+PHP_CONFIG_PREFIX = @PHP_CONFIG_PREFIX@
+PHP_PREFIX = @PHP_PREFIX@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+PYTHON = @PYTHON@
+PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@
+PYTHON_PLATFORM = @PYTHON_PLATFORM@
+PYTHON_PREFIX = @PYTHON_PREFIX@
+PYTHON_VERSION = @PYTHON_VERSION@
+PY_PREFIX = @PY_PREFIX@
+QT_CFLAGS = @QT_CFLAGS@
+QT_LIBS = @QT_LIBS@
+QT_MOC = @QT_MOC@
+RAKE = @RAKE@
+RANLIB = @RANLIB@
+RUBY = @RUBY@
+RUBY_PREFIX = @RUBY_PREFIX@
+RUNHASKELL = @RUNHASKELL@
+SED = @SED@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+STRIP = @STRIP@
+TRIAL = @TRIAL@
+VERSION = @VERSION@
+YACC = @YACC@
+YFLAGS = @YFLAGS@
+ZLIB_CPPFLAGS = @ZLIB_CPPFLAGS@
+ZLIB_LDFLAGS = @ZLIB_LDFLAGS@
+ZLIB_LIBS = @ZLIB_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+pkgpyexecdir = @pkgpyexecdir@
+pkgpythondir = @pkgpythondir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+pyexecdir = @pyexecdir@
+pythondir = @pythondir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+srcdir = @srcdir@
+subdirs = @subdirs@
+sysconfdir = @sysconfdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+EXTRA_DIST = setup.py setup.cfg src compat
+all: all-am
+
+.SUFFIXES:
+$(srcdir)/Makefile.in:  $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign lib/py/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign lib/py/Makefile
+.PRECIOUS: Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure:  $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4):  $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+tags: TAGS
+TAGS:
+
+ctags: CTAGS
+CTAGS:
+
+
+distdir: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+	$(MAKE) $(AM_MAKEFLAGS) check-local
+check: check-am
+all-am: Makefile all-local
+installdirs:
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-generic clean-libtool clean-local mostlyclean-am
+
+distclean: distclean-am
+	-rm -f Makefile
+distclean-am: clean-am distclean-generic
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am:
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+	@$(NORMAL_INSTALL)
+	$(MAKE) $(AM_MAKEFLAGS) install-exec-hook
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-generic mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am:
+
+.MAKE: check-am install-am install-exec-am install-strip
+
+.PHONY: all all-am all-local check check-am check-local clean \
+	clean-generic clean-libtool clean-local distclean \
+	distclean-generic distclean-libtool distdir dvi dvi-am html \
+	html-am info info-am install install-am install-data \
+	install-data-am install-dvi install-dvi-am install-exec \
+	install-exec-am install-exec-hook install-html install-html-am \
+	install-info install-info-am install-man install-pdf \
+	install-pdf-am install-ps install-ps-am install-strip \
+	installcheck installcheck-am installdirs maintainer-clean \
+	maintainer-clean-generic mostlyclean mostlyclean-generic \
+	mostlyclean-libtool pdf pdf-am ps ps-am uninstall uninstall-am
+
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+.NOTPARALLEL:
+DESTDIR ?= /
+
+all-local:
+	$(PYTHON) setup.py build
+
+# We're ignoring prefix here because site-packages seems to be
+# the equivalent of /usr/local/lib in Python land.
+# Old version (can't put inline because it's not portable).
+#$(PYTHON) setup.py install --prefix=$(prefix) --root=$(DESTDIR) $(PYTHON_SETUPUTIL_ARGS)
+install-exec-hook:
+	$(PYTHON) setup.py install --root=$(DESTDIR) --prefix=$(PY_PREFIX) $(PYTHON_SETUPUTIL_ARGS)
+
+clean-local:
+	$(RM) -r build
+
+check-local: all
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:

+ 0 - 0
desktop/core/ext-py/thrift/README → desktop/core/ext-py/thrift-0.9.1/README


+ 247 - 0
desktop/core/ext-py/thrift-0.9.1/compat/win32/stdint.h

@@ -0,0 +1,247 @@
+// ISO C9x  compliant stdint.h for Microsoft Visual Studio
+// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 
+// 
+//  Copyright (c) 2006-2008 Alexander Chemeris
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// 
+//   1. Redistributions of source code must retain the above copyright notice,
+//      this list of conditions and the following disclaimer.
+// 
+//   2. Redistributions in binary form must reproduce the above copyright
+//      notice, this list of conditions and the following disclaimer in the
+//      documentation and/or other materials provided with the distribution.
+// 
+//   3. The name of the author may be used to endorse or promote products
+//      derived from this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
+// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef _MSC_VER // [
+#error "Use this header only with Microsoft Visual C++ compilers!"
+#endif // _MSC_VER ]
+
+#ifndef _MSC_STDINT_H_ // [
+#define _MSC_STDINT_H_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif
+
+#include <limits.h>
+
+// For Visual Studio 6 in C++ mode and for many Visual Studio versions when
+// compiling for ARM we should wrap <wchar.h> include with 'extern "C++" {}'
+// or compiler give many errors like this:
+//   error C2733: second C linkage of overloaded function 'wmemchr' not allowed
+#ifdef __cplusplus
+extern "C" {
+#endif
+#  include <wchar.h>
+#ifdef __cplusplus
+}
+#endif
+
+// Define _W64 macros to mark types changing their size, like intptr_t.
+#ifndef _W64
+#  if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300
+#     define _W64 __w64
+#  else
+#     define _W64
+#  endif
+#endif
+
+
+// 7.18.1 Integer types
+
+// 7.18.1.1 Exact-width integer types
+
+// Visual Studio 6 and Embedded Visual C++ 4 doesn't
+// realize that, e.g. char has the same size as __int8
+// so we give up on __intX for them.
+#if (_MSC_VER < 1300)
+   typedef signed char       int8_t;
+   typedef signed short      int16_t;
+   typedef signed int        int32_t;
+   typedef unsigned char     uint8_t;
+   typedef unsigned short    uint16_t;
+   typedef unsigned int      uint32_t;
+#else
+   typedef signed __int8     int8_t;
+   typedef signed __int16    int16_t;
+   typedef signed __int32    int32_t;
+   typedef unsigned __int8   uint8_t;
+   typedef unsigned __int16  uint16_t;
+   typedef unsigned __int32  uint32_t;
+#endif
+typedef signed __int64       int64_t;
+typedef unsigned __int64     uint64_t;
+
+
+// 7.18.1.2 Minimum-width integer types
+typedef int8_t    int_least8_t;
+typedef int16_t   int_least16_t;
+typedef int32_t   int_least32_t;
+typedef int64_t   int_least64_t;
+typedef uint8_t   uint_least8_t;
+typedef uint16_t  uint_least16_t;
+typedef uint32_t  uint_least32_t;
+typedef uint64_t  uint_least64_t;
+
+// 7.18.1.3 Fastest minimum-width integer types
+typedef int8_t    int_fast8_t;
+typedef int16_t   int_fast16_t;
+typedef int32_t   int_fast32_t;
+typedef int64_t   int_fast64_t;
+typedef uint8_t   uint_fast8_t;
+typedef uint16_t  uint_fast16_t;
+typedef uint32_t  uint_fast32_t;
+typedef uint64_t  uint_fast64_t;
+
+// 7.18.1.4 Integer types capable of holding object pointers
+#ifdef _WIN64 // [
+   typedef signed __int64    intptr_t;
+   typedef unsigned __int64  uintptr_t;
+#else // _WIN64 ][
+   typedef _W64 signed int   intptr_t;
+   typedef _W64 unsigned int uintptr_t;
+#endif // _WIN64 ]
+
+// 7.18.1.5 Greatest-width integer types
+typedef int64_t   intmax_t;
+typedef uint64_t  uintmax_t;
+
+
+// 7.18.2 Limits of specified-width integer types
+
+#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [   See footnote 220 at page 257 and footnote 221 at page 259
+
+// 7.18.2.1 Limits of exact-width integer types
+#define INT8_MIN     ((int8_t)_I8_MIN)
+#define INT8_MAX     _I8_MAX
+#define INT16_MIN    ((int16_t)_I16_MIN)
+#define INT16_MAX    _I16_MAX
+#define INT32_MIN    ((int32_t)_I32_MIN)
+#define INT32_MAX    _I32_MAX
+#define INT64_MIN    ((int64_t)_I64_MIN)
+#define INT64_MAX    _I64_MAX
+#define UINT8_MAX    _UI8_MAX
+#define UINT16_MAX   _UI16_MAX
+#define UINT32_MAX   _UI32_MAX
+#define UINT64_MAX   _UI64_MAX
+
+// 7.18.2.2 Limits of minimum-width integer types
+#define INT_LEAST8_MIN    INT8_MIN
+#define INT_LEAST8_MAX    INT8_MAX
+#define INT_LEAST16_MIN   INT16_MIN
+#define INT_LEAST16_MAX   INT16_MAX
+#define INT_LEAST32_MIN   INT32_MIN
+#define INT_LEAST32_MAX   INT32_MAX
+#define INT_LEAST64_MIN   INT64_MIN
+#define INT_LEAST64_MAX   INT64_MAX
+#define UINT_LEAST8_MAX   UINT8_MAX
+#define UINT_LEAST16_MAX  UINT16_MAX
+#define UINT_LEAST32_MAX  UINT32_MAX
+#define UINT_LEAST64_MAX  UINT64_MAX
+
+// 7.18.2.3 Limits of fastest minimum-width integer types
+#define INT_FAST8_MIN    INT8_MIN
+#define INT_FAST8_MAX    INT8_MAX
+#define INT_FAST16_MIN   INT16_MIN
+#define INT_FAST16_MAX   INT16_MAX
+#define INT_FAST32_MIN   INT32_MIN
+#define INT_FAST32_MAX   INT32_MAX
+#define INT_FAST64_MIN   INT64_MIN
+#define INT_FAST64_MAX   INT64_MAX
+#define UINT_FAST8_MAX   UINT8_MAX
+#define UINT_FAST16_MAX  UINT16_MAX
+#define UINT_FAST32_MAX  UINT32_MAX
+#define UINT_FAST64_MAX  UINT64_MAX
+
+// 7.18.2.4 Limits of integer types capable of holding object pointers
+#ifdef _WIN64 // [
+#  define INTPTR_MIN   INT64_MIN
+#  define INTPTR_MAX   INT64_MAX
+#  define UINTPTR_MAX  UINT64_MAX
+#else // _WIN64 ][
+#  define INTPTR_MIN   INT32_MIN
+#  define INTPTR_MAX   INT32_MAX
+#  define UINTPTR_MAX  UINT32_MAX
+#endif // _WIN64 ]
+
+// 7.18.2.5 Limits of greatest-width integer types
+#define INTMAX_MIN   INT64_MIN
+#define INTMAX_MAX   INT64_MAX
+#define UINTMAX_MAX  UINT64_MAX
+
+// 7.18.3 Limits of other integer types
+
+#ifdef _WIN64 // [
+#  define PTRDIFF_MIN  _I64_MIN
+#  define PTRDIFF_MAX  _I64_MAX
+#else  // _WIN64 ][
+#  define PTRDIFF_MIN  _I32_MIN
+#  define PTRDIFF_MAX  _I32_MAX
+#endif  // _WIN64 ]
+
+#define SIG_ATOMIC_MIN  INT_MIN
+#define SIG_ATOMIC_MAX  INT_MAX
+
+#ifndef SIZE_MAX // [
+#  ifdef _WIN64 // [
+#     define SIZE_MAX  _UI64_MAX
+#  else // _WIN64 ][
+#     define SIZE_MAX  _UI32_MAX
+#  endif // _WIN64 ]
+#endif // SIZE_MAX ]
+
+// WCHAR_MIN and WCHAR_MAX are also defined in <wchar.h>
+#ifndef WCHAR_MIN // [
+#  define WCHAR_MIN  0
+#endif  // WCHAR_MIN ]
+#ifndef WCHAR_MAX // [
+#  define WCHAR_MAX  _UI16_MAX
+#endif  // WCHAR_MAX ]
+
+#define WINT_MIN  0
+#define WINT_MAX  _UI16_MAX
+
+#endif // __STDC_LIMIT_MACROS ]
+
+
+// 7.18.4 Limits of other integer types
+
+#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [   See footnote 224 at page 260
+
+// 7.18.4.1 Macros for minimum-width integer constants
+
+#define INT8_C(val)  val##i8
+#define INT16_C(val) val##i16
+#define INT32_C(val) val##i32
+#define INT64_C(val) val##i64
+
+#define UINT8_C(val)  val##ui8
+#define UINT16_C(val) val##ui16
+#define UINT32_C(val) val##ui32
+#define UINT64_C(val) val##ui64
+
+// 7.18.4.2 Macros for greatest-width integer constants
+#define INTMAX_C   INT64_C
+#define UINTMAX_C  UINT64_C
+
+#endif // __STDC_CONSTANT_MACROS ]
+
+
+#endif // _MSC_STDINT_H_ ]

+ 2 - 0
desktop/core/ext-py/thrift-0.9.1/setup.cfg

@@ -0,0 +1,2 @@
+[install]
+optimize = 1

+ 104 - 0
desktop/core/ext-py/thrift-0.9.1/setup.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import sys
+try:
+    from setuptools import setup, Extension
+except:
+    from distutils.core import setup, Extension, Command
+
+from distutils.command.build_ext import build_ext
+from distutils.errors import CCompilerError, DistutilsExecError, DistutilsPlatformError
+
+include_dirs = []
+if sys.platform == 'win32':
+    include_dirs.append('compat/win32')
+    ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError, IOError)
+else:
+    ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError)
+
+class BuildFailed(Exception):
+    pass
+
+class ve_build_ext(build_ext):
+    def run(self):
+        try:
+            build_ext.run(self)
+        except DistutilsPlatformError, x:
+            raise BuildFailed()
+
+    def build_extension(self, ext):
+        try:
+            build_ext.build_extension(self, ext)
+        except ext_errors, x:
+            raise BuildFailed()
+
+def run_setup(with_binary):
+    if with_binary:
+        extensions = dict(
+            ext_modules = [
+                 Extension('thrift.protocol.fastbinary',
+                       sources = ['src/protocol/fastbinary.c'],
+                    include_dirs = include_dirs,
+                )
+            ],
+            cmdclass=dict(build_ext=ve_build_ext)
+        )
+    else:
+        extensions = dict()
+        
+    setup(name = 'thrift',
+        version = '0.9.1',
+        description = 'Python bindings for the Apache Thrift RPC system',
+        author = ['Thrift Developers'],
+        author_email = ['dev@thrift.apache.org'],
+        url = 'http://thrift.apache.org',
+        license = 'Apache License 2.0',
+        packages = [
+            'thrift',
+            'thrift.protocol',
+            'thrift.transport',
+            'thrift.server',
+        ],
+        package_dir = {'thrift' : 'src'},
+        classifiers = [
+            'Development Status :: 5 - Production/Stable',
+            'Environment :: Console',
+            'Intended Audience :: Developers',
+            'Programming Language :: Python',
+            'Programming Language :: Python :: 2',
+            'Topic :: Software Development :: Libraries',
+            'Topic :: System :: Networking'
+        ],
+        **extensions
+    )
+
+try:
+    run_setup(True)
+except BuildFailed:
+    print
+    print '*' * 80
+    print "An error occured while trying to compile with the C extension enabled" 
+    print "Attempting to build without the extension now"
+    print '*' * 80
+    print
+
+    run_setup(False)

+ 5 - 3
desktop/core/ext-py/thrift/src/TSCons.py → desktop/core/ext-py/thrift-0.9.1/src/TSCons.py

@@ -20,14 +20,16 @@
 from os import path
 from SCons.Builder import Builder
 
+
 def scons_env(env, add=''):
   opath = path.dirname(path.abspath('$TARGET'))
   lstr = 'thrift --gen cpp -o ' + opath + ' ' + add + ' $SOURCE'
-  cppbuild = Builder(action = lstr)
-  env.Append(BUILDERS = {'ThriftCpp' : cppbuild})
+  cppbuild = Builder(action=lstr)
+  env.Append(BUILDERS={'ThriftCpp': cppbuild})
+
 
 def gen_cpp(env, dir, file):
   scons_env(env)
   suffixes = ['_types.h', '_types.cpp']
   targets = map(lambda s: 'gen-cpp/' + file + s, suffixes)
-  return env.ThriftCpp(targets, dir+file+'.thrift')
+  return env.ThriftCpp(targets, dir + file + '.thrift')

+ 16 - 22
desktop/core/ext-py/thrift/setup.py → desktop/core/ext-py/thrift-0.9.1/src/TSerialization.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements. See the NOTICE file
@@ -19,26 +17,22 @@
 # under the License.
 #
 
-from distutils.core import setup, Extension
+from protocol import TBinaryProtocol
+from transport import TTransport
+
 
-fastbinarymod = Extension('thrift.protocol.fastbinary',
-                          sources = ['src/protocol/fastbinary.c'],
-                          )
+def serialize(thrift_object,
+              protocol_factory=TBinaryProtocol.TBinaryProtocolFactory()):
+    transport = TTransport.TMemoryBuffer()
+    protocol = protocol_factory.getProtocol(transport)
+    thrift_object.write(protocol)
+    return transport.getvalue()
 
-setup(name = 'Thrift',
-      version = '0.1',
-      description = 'Thrift Python Libraries',
-      author = ['Thrift Developers'],
-      author_email = ['thrift-dev@incubator.apache.org'],
-      url = 'http://incubator.apache.org/thrift/',
-      license = 'Apache License 2.0',
-      packages = [
-        'thrift',
-        'thrift.protocol',
-        'thrift.transport',
-        'thrift.server',
-      ],
-      package_dir = {'thrift' : 'src'},
-      ext_modules = [fastbinarymod],
-      )
 
+def deserialize(base,
+                buf,
+                protocol_factory=TBinaryProtocol.TBinaryProtocolFactory()):
+    transport = TTransport.TMemoryBuffer(buf)
+    protocol = protocol_factory.getProtocol(transport)
+    base.read(protocol)
+    return base

+ 153 - 0
desktop/core/ext-py/thrift-0.9.1/src/TTornado.py

@@ -0,0 +1,153 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from cStringIO import StringIO
+import logging
+import socket
+import struct
+
+from thrift.transport import TTransport
+from thrift.transport.TTransport import TTransportException
+
+from tornado import gen
+from tornado import iostream
+from tornado import netutil
+
+
+class TTornadoStreamTransport(TTransport.TTransportBase):
+    """a framed, buffered transport over a Tornado stream"""
+    def __init__(self, host, port, stream=None):
+        self.host = host
+        self.port = port
+        self.is_queuing_reads = False
+        self.read_queue = []
+        self.__wbuf = StringIO()
+
+        # servers provide a ready-to-go stream
+        self.stream = stream
+        if self.stream is not None:
+            self._set_close_callback()
+
+    # not the same number of parameters as TTransportBase.open
+    def open(self, callback):
+        logging.debug('socket connecting')
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
+        self.stream = iostream.IOStream(sock)
+
+        def on_close_in_connect(*_):
+            message = 'could not connect to {}:{}'.format(self.host, self.port)
+            raise TTransportException(
+                type=TTransportException.NOT_OPEN,
+                message=message)
+        self.stream.set_close_callback(on_close_in_connect)
+
+        def finish(*_):
+            self._set_close_callback()
+            callback()
+
+        self.stream.connect((self.host, self.port), callback=finish)
+
+    def _set_close_callback(self):
+        def on_close():
+            raise TTransportException(
+                type=TTransportException.END_OF_FILE,
+                message='socket closed')
+        self.stream.set_close_callback(self.close)
+
+    def close(self):
+        # don't raise if we intend to close
+        self.stream.set_close_callback(None)
+        self.stream.close()
+
+    def read(self, _):
+        # The generated code for Tornado shouldn't do individual reads -- only
+        # frames at a time
+        assert "you're doing it wrong" is True
+
+    @gen.engine
+    def readFrame(self, callback):
+        self.read_queue.append(callback)
+        logging.debug('read queue: %s', self.read_queue)
+
+        if self.is_queuing_reads:
+            # If a read is already in flight, then the while loop below should
+            # pull it from self.read_queue
+            return
+
+        self.is_queuing_reads = True
+        while self.read_queue:
+            next_callback = self.read_queue.pop()
+            result = yield gen.Task(self._readFrameFromStream)
+            next_callback(result)
+        self.is_queuing_reads = False
+
+    @gen.engine
+    def _readFrameFromStream(self, callback):
+        logging.debug('_readFrameFromStream')
+        frame_header = yield gen.Task(self.stream.read_bytes, 4)
+        frame_length, = struct.unpack('!i', frame_header)
+        logging.debug('received frame header, frame length = %i', frame_length)
+        frame = yield gen.Task(self.stream.read_bytes, frame_length)
+        logging.debug('received frame payload')
+        callback(frame)
+
+    def write(self, buf):
+        self.__wbuf.write(buf)
+
+    def flush(self, callback=None):
+        wout = self.__wbuf.getvalue()
+        wsz = len(wout)
+        # reset wbuf before write/flush to preserve state on underlying failure
+        self.__wbuf = StringIO()
+        # N.B.: Doing this string concatenation is WAY cheaper than making
+        # two separate calls to the underlying socket object. Socket writes in
+        # Python turn out to be REALLY expensive, but it seems to do a pretty
+        # good job of managing string buffer operations without excessive copies
+        buf = struct.pack("!i", wsz) + wout
+
+        logging.debug('writing frame length = %i', wsz)
+        self.stream.write(buf, callback)
+
+
+class TTornadoServer(netutil.TCPServer):
+    def __init__(self, processor, iprot_factory, oprot_factory=None,
+                 *args, **kwargs):
+        super(TTornadoServer, self).__init__(*args, **kwargs)
+
+        self._processor = processor
+        self._iprot_factory = iprot_factory
+        self._oprot_factory = (oprot_factory if oprot_factory is not None
+                               else iprot_factory)
+
+    def handle_stream(self, stream, address):
+        try:
+            host, port = address
+            trans = TTornadoStreamTransport(host=host, port=port, stream=stream)
+            oprot = self._oprot_factory.getProtocol(trans)
+
+            def next_pass():
+                if not trans.stream.closed():
+                    self._processor.process(trans, self._iprot_factory, oprot,
+                                            callback=next_pass)
+
+            next_pass()
+
+        except Exception:
+            logging.exception('thrift exception in handle_stream')
+            trans.close()

+ 60 - 13
desktop/core/ext-py/thrift/src/Thrift.py → desktop/core/ext-py/thrift-0.9.1/src/Thrift.py

@@ -17,6 +17,9 @@
 # under the License.
 #
 
+import sys
+
+
 class TType:
   STOP   = 0
   VOID   = 1
@@ -36,29 +39,58 @@ class TType:
   UTF8   = 16
   UTF16  = 17
 
+  _VALUES_TO_NAMES = ('STOP',
+                      'VOID',
+                      'BOOL',
+                      'BYTE',
+                      'DOUBLE',
+                      None,
+                      'I16',
+                      None,
+                      'I32',
+                      None,
+                     'I64',
+                     'STRING',
+                     'STRUCT',
+                     'MAP',
+                     'SET',
+                     'LIST',
+                     'UTF8',
+                     'UTF16')
+
+
 class TMessageType:
-  CALL  = 1
+  CALL = 1
   REPLY = 2
   EXCEPTION = 3
   ONEWAY = 4
 
-class TProcessor:
 
+class TProcessor:
   """Base class for procsessor, which works on two streams."""
 
   def process(iprot, oprot):
     pass
 
-class TException(Exception):
 
+class TException(Exception):
   """Base class for all thrift exceptions."""
 
+  # BaseException.message is deprecated in Python v[2.6,3.0)
+  if (2, 6, 0) <= sys.version_info < (3, 0):
+    def _get_message(self):
+      return self._message
+
+    def _set_message(self, message):
+      self._message = message
+    message = property(_get_message, _set_message)
+
   def __init__(self, message=None):
     Exception.__init__(self, message)
     self.message = message
 
-class TApplicationException(TException):
 
+class TApplicationException(TException):
   """Application level thrift exceptions."""
 
   UNKNOWN = 0
@@ -67,6 +99,11 @@ class TApplicationException(TException):
   WRONG_METHOD_NAME = 3
   BAD_SEQUENCE_ID = 4
   MISSING_RESULT = 5
+  INTERNAL_ERROR = 6
+  PROTOCOL_ERROR = 7
+  INVALID_TRANSFORM = 8
+  INVALID_PROTOCOL = 9
+  UNSUPPORTED_CLIENT_TYPE = 10
 
   def __init__(self, type=UNKNOWN, message=None):
     TException.__init__(self, message)
@@ -75,16 +112,26 @@ class TApplicationException(TException):
   def __str__(self):
     if self.message:
       return self.message
-    elif self.type == UNKNOWN_METHOD:
+    elif self.type == self.UNKNOWN_METHOD:
       return 'Unknown method'
-    elif self.type == INVALID_MESSAGE_TYPE:
+    elif self.type == self.INVALID_MESSAGE_TYPE:
       return 'Invalid message type'
-    elif self.type == WRONG_METHOD_NAME:
+    elif self.type == self.WRONG_METHOD_NAME:
       return 'Wrong method name'
-    elif self.type == BAD_SEQUENCE_ID:
+    elif self.type == self.BAD_SEQUENCE_ID:
       return 'Bad sequence ID'
-    elif self.type == MISSING_RESULT:
+    elif self.type == self.MISSING_RESULT:
       return 'Missing result'
+    elif self.type == self.INTERNAL_ERROR:
+      return 'Internal error'
+    elif self.type == self.PROTOCOL_ERROR:
+      return 'Protocol error'
+    elif self.type == self.INVALID_TRANSFORM:
+      return 'Invalid transform'
+    elif self.type == self.INVALID_PROTOCOL:
+      return 'Invalid protocol'
+    elif self.type == self.UNSUPPORTED_CLIENT_TYPE:
+      return 'Unsupported client type'
     else:
       return 'Default (unknown) TApplicationException'
 
@@ -96,12 +143,12 @@ class TApplicationException(TException):
         break
       if fid == 1:
         if ftype == TType.STRING:
-          self.message = iprot.readString();
+          self.message = iprot.readString()
         else:
           iprot.skip(ftype)
       elif fid == 2:
         if ftype == TType.I32:
-          self.type = iprot.readI32();
+          self.type = iprot.readI32()
         else:
           iprot.skip(ftype)
       else:
@@ -111,11 +158,11 @@ class TApplicationException(TException):
 
   def write(self, oprot):
     oprot.writeStructBegin('TApplicationException')
-    if self.message != None:
+    if self.message is not None:
       oprot.writeFieldBegin('message', TType.STRING, 1)
       oprot.writeString(self.message)
       oprot.writeFieldEnd()
-    if self.type != None:
+    if self.type is not None:
       oprot.writeFieldBegin('type', TType.I32, 2)
       oprot.writeI32(self.type)
       oprot.writeFieldEnd()

+ 0 - 0
desktop/core/ext-py/thrift/src/__init__.py → desktop/core/ext-py/thrift-0.9.1/src/__init__.py


+ 81 - 0
desktop/core/ext-py/thrift-0.9.1/src/protocol/TBase.py

@@ -0,0 +1,81 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from thrift.Thrift import *
+from thrift.protocol import TBinaryProtocol
+from thrift.transport import TTransport
+
+try:
+  from thrift.protocol import fastbinary
+except:
+  fastbinary = None
+
+
+class TBase(object):
+  __slots__ = []
+
+  def __repr__(self):
+    L = ['%s=%r' % (key, getattr(self, key))
+              for key in self.__slots__]
+    return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+
+  def __eq__(self, other):
+    if not isinstance(other, self.__class__):
+      return False
+    for attr in self.__slots__:
+      my_val = getattr(self, attr)
+      other_val = getattr(other, attr)
+      if my_val != other_val:
+        return False
+    return True
+
+  def __ne__(self, other):
+    return not (self == other)
+
+  def read(self, iprot):
+    if (iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and
+        isinstance(iprot.trans, TTransport.CReadableTransport) and
+        self.thrift_spec is not None and
+        fastbinary is not None):
+      fastbinary.decode_binary(self,
+                               iprot.trans,
+                               (self.__class__, self.thrift_spec))
+      return
+    iprot.readStruct(self, self.thrift_spec)
+
+  def write(self, oprot):
+    if (oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and
+        self.thrift_spec is not None and
+        fastbinary is not None):
+      oprot.trans.write(
+        fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+      return
+    oprot.writeStruct(self, self.thrift_spec)
+
+
+class TExceptionBase(Exception):
+  # old style class so python2.4 can raise exceptions derived from this
+  #  This can't inherit from TBase because of that limitation.
+  __slots__ = []
+
+  __repr__ = TBase.__repr__.im_func
+  __eq__ = TBase.__eq__.im_func
+  __ne__ = TBase.__ne__.im_func
+  read = TBase.read.im_func
+  write = TBase.write.im_func

+ 7 - 6
desktop/core/ext-py/thrift/src/protocol/TBinaryProtocol.py → desktop/core/ext-py/thrift-0.9.1/src/protocol/TBinaryProtocol.py

@@ -20,8 +20,8 @@
 from TProtocol import *
 from struct import pack, unpack
 
-class TBinaryProtocol(TProtocolBase):
 
+class TBinaryProtocol(TProtocolBase):
   """Binary implementation of the Thrift protocol driver."""
 
   # NastyHaxx. Python 2.4+ on 32-bit machines forces hex constants to be
@@ -68,7 +68,7 @@ class TBinaryProtocol(TProtocolBase):
     pass
 
   def writeFieldStop(self):
-    self.writeByte(TType.STOP);
+    self.writeByte(TType.STOP)
 
   def writeMapBegin(self, ktype, vtype, size):
     self.writeByte(ktype)
@@ -127,13 +127,16 @@ class TBinaryProtocol(TProtocolBase):
     if sz < 0:
       version = sz & TBinaryProtocol.VERSION_MASK
       if version != TBinaryProtocol.VERSION_1:
-        raise TProtocolException(TProtocolException.BAD_VERSION, 'Bad version in readMessageBegin: %d' % (sz))
+        raise TProtocolException(
+          type=TProtocolException.BAD_VERSION,
+          message='Bad version in readMessageBegin: %d' % (sz))
       type = sz & TBinaryProtocol.TYPE_MASK
       name = self.readString()
       seqid = self.readI32()
     else:
       if self.strictRead:
-        raise TProtocolException(TProtocolException.BAD_VERSION, 'No protocol version header')
+        raise TProtocolException(type=TProtocolException.BAD_VERSION,
+                                 message='No protocol version header')
       name = self.trans.readAll(sz)
       type = self.readByte()
       seqid = self.readI32()
@@ -231,7 +234,6 @@ class TBinaryProtocolFactory:
 
 
 class TBinaryProtocolAccelerated(TBinaryProtocol):
-
   """C-Accelerated version of TBinaryProtocol.
 
   This class does not override any of TBinaryProtocol's methods,
@@ -250,7 +252,6 @@ class TBinaryProtocolAccelerated(TBinaryProtocol):
          Please feel free to report bugs and/or success stories
          to the public mailing list.
   """
-
   pass
 
 

+ 403 - 0
desktop/core/ext-py/thrift-0.9.1/src/protocol/TCompactProtocol.py

@@ -0,0 +1,403 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from TProtocol import *
+from struct import pack, unpack
+
+__all__ = ['TCompactProtocol', 'TCompactProtocolFactory']
+
+CLEAR = 0
+FIELD_WRITE = 1
+VALUE_WRITE = 2
+CONTAINER_WRITE = 3
+BOOL_WRITE = 4
+FIELD_READ = 5
+CONTAINER_READ = 6
+VALUE_READ = 7
+BOOL_READ = 8
+
+
+def make_helper(v_from, container):
+  def helper(func):
+    def nested(self, *args, **kwargs):
+      assert self.state in (v_from, container), (self.state, v_from, container)
+      return func(self, *args, **kwargs)
+    return nested
+  return helper
+writer = make_helper(VALUE_WRITE, CONTAINER_WRITE)
+reader = make_helper(VALUE_READ, CONTAINER_READ)
+
+
+def makeZigZag(n, bits):
+  return (n << 1) ^ (n >> (bits - 1))
+
+
+def fromZigZag(n):
+  return (n >> 1) ^ -(n & 1)
+
+
+def writeVarint(trans, n):
+  out = []
+  while True:
+    if n & ~0x7f == 0:
+      out.append(n)
+      break
+    else:
+      out.append((n & 0xff) | 0x80)
+      n = n >> 7
+  trans.write(''.join(map(chr, out)))
+
+
+def readVarint(trans):
+  result = 0
+  shift = 0
+  while True:
+    x = trans.readAll(1)
+    byte = ord(x)
+    result |= (byte & 0x7f) << shift
+    if byte >> 7 == 0:
+      return result
+    shift += 7
+
+
+class CompactType:
+  STOP = 0x00
+  TRUE = 0x01
+  FALSE = 0x02
+  BYTE = 0x03
+  I16 = 0x04
+  I32 = 0x05
+  I64 = 0x06
+  DOUBLE = 0x07
+  BINARY = 0x08
+  LIST = 0x09
+  SET = 0x0A
+  MAP = 0x0B
+  STRUCT = 0x0C
+
+CTYPES = {TType.STOP: CompactType.STOP,
+          TType.BOOL: CompactType.TRUE,  # used for collection
+          TType.BYTE: CompactType.BYTE,
+          TType.I16: CompactType.I16,
+          TType.I32: CompactType.I32,
+          TType.I64: CompactType.I64,
+          TType.DOUBLE: CompactType.DOUBLE,
+          TType.STRING: CompactType.BINARY,
+          TType.STRUCT: CompactType.STRUCT,
+          TType.LIST: CompactType.LIST,
+          TType.SET: CompactType.SET,
+          TType.MAP: CompactType.MAP
+          }
+
+TTYPES = {}
+for k, v in CTYPES.items():
+  TTYPES[v] = k
+TTYPES[CompactType.FALSE] = TType.BOOL
+del k
+del v
+
+
+class TCompactProtocol(TProtocolBase):
+  """Compact implementation of the Thrift protocol driver."""
+
+  PROTOCOL_ID = 0x82
+  VERSION = 1
+  VERSION_MASK = 0x1f
+  TYPE_MASK = 0xe0
+  TYPE_SHIFT_AMOUNT = 5
+
+  def __init__(self, trans):
+    TProtocolBase.__init__(self, trans)
+    self.state = CLEAR
+    self.__last_fid = 0
+    self.__bool_fid = None
+    self.__bool_value = None
+    self.__structs = []
+    self.__containers = []
+
+  def __writeVarint(self, n):
+    writeVarint(self.trans, n)
+
+  def writeMessageBegin(self, name, type, seqid):
+    assert self.state == CLEAR
+    self.__writeUByte(self.PROTOCOL_ID)
+    self.__writeUByte(self.VERSION | (type << self.TYPE_SHIFT_AMOUNT))
+    self.__writeVarint(seqid)
+    self.__writeString(name)
+    self.state = VALUE_WRITE
+
+  def writeMessageEnd(self):
+    assert self.state == VALUE_WRITE
+    self.state = CLEAR
+
+  def writeStructBegin(self, name):
+    assert self.state in (CLEAR, CONTAINER_WRITE, VALUE_WRITE), self.state
+    self.__structs.append((self.state, self.__last_fid))
+    self.state = FIELD_WRITE
+    self.__last_fid = 0
+
+  def writeStructEnd(self):
+    assert self.state == FIELD_WRITE
+    self.state, self.__last_fid = self.__structs.pop()
+
+  def writeFieldStop(self):
+    self.__writeByte(0)
+
+  def __writeFieldHeader(self, type, fid):
+    delta = fid - self.__last_fid
+    if 0 < delta <= 15:
+      self.__writeUByte(delta << 4 | type)
+    else:
+      self.__writeByte(type)
+      self.__writeI16(fid)
+    self.__last_fid = fid
+
+  def writeFieldBegin(self, name, type, fid):
+    assert self.state == FIELD_WRITE, self.state
+    if type == TType.BOOL:
+      self.state = BOOL_WRITE
+      self.__bool_fid = fid
+    else:
+      self.state = VALUE_WRITE
+      self.__writeFieldHeader(CTYPES[type], fid)
+
+  def writeFieldEnd(self):
+    assert self.state in (VALUE_WRITE, BOOL_WRITE), self.state
+    self.state = FIELD_WRITE
+
+  def __writeUByte(self, byte):
+    self.trans.write(pack('!B', byte))
+
+  def __writeByte(self, byte):
+    self.trans.write(pack('!b', byte))
+
+  def __writeI16(self, i16):
+    self.__writeVarint(makeZigZag(i16, 16))
+
+  def __writeSize(self, i32):
+    self.__writeVarint(i32)
+
+  def writeCollectionBegin(self, etype, size):
+    assert self.state in (VALUE_WRITE, CONTAINER_WRITE), self.state
+    if size <= 14:
+      self.__writeUByte(size << 4 | CTYPES[etype])
+    else:
+      self.__writeUByte(0xf0 | CTYPES[etype])
+      self.__writeSize(size)
+    self.__containers.append(self.state)
+    self.state = CONTAINER_WRITE
+  writeSetBegin = writeCollectionBegin
+  writeListBegin = writeCollectionBegin
+
+  def writeMapBegin(self, ktype, vtype, size):
+    assert self.state in (VALUE_WRITE, CONTAINER_WRITE), self.state
+    if size == 0:
+      self.__writeByte(0)
+    else:
+      self.__writeSize(size)
+      self.__writeUByte(CTYPES[ktype] << 4 | CTYPES[vtype])
+    self.__containers.append(self.state)
+    self.state = CONTAINER_WRITE
+
+  def writeCollectionEnd(self):
+    assert self.state == CONTAINER_WRITE, self.state
+    self.state = self.__containers.pop()
+  writeMapEnd = writeCollectionEnd
+  writeSetEnd = writeCollectionEnd
+  writeListEnd = writeCollectionEnd
+
+  def writeBool(self, bool):
+    if self.state == BOOL_WRITE:
+      if bool:
+        ctype = CompactType.TRUE
+      else:
+        ctype = CompactType.FALSE
+      self.__writeFieldHeader(ctype, self.__bool_fid)
+    elif self.state == CONTAINER_WRITE:
+      if bool:
+        self.__writeByte(CompactType.TRUE)
+      else:
+        self.__writeByte(CompactType.FALSE)
+    else:
+      raise AssertionError("Invalid state in compact protocol")
+
+  writeByte = writer(__writeByte)
+  writeI16 = writer(__writeI16)
+
+  @writer
+  def writeI32(self, i32):
+    self.__writeVarint(makeZigZag(i32, 32))
+
+  @writer
+  def writeI64(self, i64):
+    self.__writeVarint(makeZigZag(i64, 64))
+
+  @writer
+  def writeDouble(self, dub):
+    self.trans.write(pack('!d', dub))
+
+  def __writeString(self, s):
+    self.__writeSize(len(s))
+    self.trans.write(s)
+  writeString = writer(__writeString)
+
+  def readFieldBegin(self):
+    assert self.state == FIELD_READ, self.state
+    type = self.__readUByte()
+    if type & 0x0f == TType.STOP:
+      return (None, 0, 0)
+    delta = type >> 4
+    if delta == 0:
+      fid = self.__readI16()
+    else:
+      fid = self.__last_fid + delta
+    self.__last_fid = fid
+    type = type & 0x0f
+    if type == CompactType.TRUE:
+      self.state = BOOL_READ
+      self.__bool_value = True
+    elif type == CompactType.FALSE:
+      self.state = BOOL_READ
+      self.__bool_value = False
+    else:
+      self.state = VALUE_READ
+    return (None, self.__getTType(type), fid)
+
+  def readFieldEnd(self):
+    assert self.state in (VALUE_READ, BOOL_READ), self.state
+    self.state = FIELD_READ
+
+  def __readUByte(self):
+    result, = unpack('!B', self.trans.readAll(1))
+    return result
+
+  def __readByte(self):
+    result, = unpack('!b', self.trans.readAll(1))
+    return result
+
+  def __readVarint(self):
+    return readVarint(self.trans)
+
+  def __readZigZag(self):
+    return fromZigZag(self.__readVarint())
+
+  def __readSize(self):
+    result = self.__readVarint()
+    if result < 0:
+      raise TException("Length < 0")
+    return result
+
+  def readMessageBegin(self):
+    assert self.state == CLEAR
+    proto_id = self.__readUByte()
+    if proto_id != self.PROTOCOL_ID:
+      raise TProtocolException(TProtocolException.BAD_VERSION,
+          'Bad protocol id in the message: %d' % proto_id)
+    ver_type = self.__readUByte()
+    type = (ver_type & self.TYPE_MASK) >> self.TYPE_SHIFT_AMOUNT
+    version = ver_type & self.VERSION_MASK
+    if version != self.VERSION:
+      raise TProtocolException(TProtocolException.BAD_VERSION,
+          'Bad version: %d (expect %d)' % (version, self.VERSION))
+    seqid = self.__readVarint()
+    name = self.__readString()
+    return (name, type, seqid)
+
+  def readMessageEnd(self):
+    assert self.state == CLEAR
+    assert len(self.__structs) == 0
+
+  def readStructBegin(self):
+    assert self.state in (CLEAR, CONTAINER_READ, VALUE_READ), self.state
+    self.__structs.append((self.state, self.__last_fid))
+    self.state = FIELD_READ
+    self.__last_fid = 0
+
+  def readStructEnd(self):
+    assert self.state == FIELD_READ
+    self.state, self.__last_fid = self.__structs.pop()
+
+  def readCollectionBegin(self):
+    assert self.state in (VALUE_READ, CONTAINER_READ), self.state
+    size_type = self.__readUByte()
+    size = size_type >> 4
+    type = self.__getTType(size_type)
+    if size == 15:
+      size = self.__readSize()
+    self.__containers.append(self.state)
+    self.state = CONTAINER_READ
+    return type, size
+  readSetBegin = readCollectionBegin
+  readListBegin = readCollectionBegin
+
+  def readMapBegin(self):
+    assert self.state in (VALUE_READ, CONTAINER_READ), self.state
+    size = self.__readSize()
+    types = 0
+    if size > 0:
+      types = self.__readUByte()
+    vtype = self.__getTType(types)
+    ktype = self.__getTType(types >> 4)
+    self.__containers.append(self.state)
+    self.state = CONTAINER_READ
+    return (ktype, vtype, size)
+
+  def readCollectionEnd(self):
+    assert self.state == CONTAINER_READ, self.state
+    self.state = self.__containers.pop()
+  readSetEnd = readCollectionEnd
+  readListEnd = readCollectionEnd
+  readMapEnd = readCollectionEnd
+
+  def readBool(self):
+    if self.state == BOOL_READ:
+      return self.__bool_value == CompactType.TRUE
+    elif self.state == CONTAINER_READ:
+      return self.__readByte() == CompactType.TRUE
+    else:
+      raise AssertionError("Invalid state in compact protocol: %d" %
+                           self.state)
+
+  readByte = reader(__readByte)
+  __readI16 = __readZigZag
+  readI16 = reader(__readZigZag)
+  readI32 = reader(__readZigZag)
+  readI64 = reader(__readZigZag)
+
+  @reader
+  def readDouble(self):
+    buff = self.trans.readAll(8)
+    val, = unpack('!d', buff)
+    return val
+
+  def __readString(self):
+    len = self.__readSize()
+    return self.trans.readAll(len)
+  readString = reader(__readString)
+
+  def __getTType(self, byte):
+    return TTYPES[byte & 0x0f]
+
+
+class TCompactProtocolFactory:
+  def __init__(self):
+    pass
+
+  def getProtocol(self, trans):
+    return TCompactProtocol(trans)

+ 550 - 0
desktop/core/ext-py/thrift-0.9.1/src/protocol/TJSONProtocol.py

@@ -0,0 +1,550 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from TProtocol import TType, TProtocolBase, TProtocolException
+import base64
+import json
+import math
+
+__all__ = ['TJSONProtocol',
+           'TJSONProtocolFactory',
+           'TSimpleJSONProtocol',
+           'TSimpleJSONProtocolFactory']
+
+VERSION = 1
+
+COMMA = ','
+COLON = ':'
+LBRACE = '{'
+RBRACE = '}'
+LBRACKET = '['
+RBRACKET = ']'
+QUOTE = '"'
+BACKSLASH = '\\'
+ZERO = '0'
+
+ESCSEQ = '\\u00'
+ESCAPE_CHAR = '"\\bfnrt'
+ESCAPE_CHAR_VALS = ['"', '\\', '\b', '\f', '\n', '\r', '\t']
+NUMERIC_CHAR = '+-.0123456789Ee'
+
+CTYPES = {TType.BOOL:       'tf',
+          TType.BYTE:       'i8',
+          TType.I16:        'i16',
+          TType.I32:        'i32',
+          TType.I64:        'i64',
+          TType.DOUBLE:     'dbl',
+          TType.STRING:     'str',
+          TType.STRUCT:     'rec',
+          TType.LIST:       'lst',
+          TType.SET:        'set',
+          TType.MAP:        'map'}
+
+JTYPES = {}
+for key in CTYPES.keys():
+  JTYPES[CTYPES[key]] = key
+
+
+class JSONBaseContext(object):
+
+  def __init__(self, protocol):
+    self.protocol = protocol
+    self.first = True
+
+  def doIO(self, function):
+    pass
+  
+  def write(self):
+    pass
+
+  def read(self):
+    pass
+
+  def escapeNum(self):
+    return False
+
+  def __str__(self):
+    return self.__class__.__name__
+
+
+class JSONListContext(JSONBaseContext):
+    
+  def doIO(self, function):
+    if self.first is True:
+      self.first = False
+    else:
+      function(COMMA)
+
+  def write(self):
+    self.doIO(self.protocol.trans.write)
+
+  def read(self):
+    self.doIO(self.protocol.readJSONSyntaxChar)
+
+
+class JSONPairContext(JSONBaseContext):
+  
+  def __init__(self, protocol):
+    super(JSONPairContext, self).__init__(protocol)
+    self.colon = True
+
+  def doIO(self, function):
+    if self.first:
+      self.first = False
+      self.colon = True
+    else:
+      function(COLON if self.colon else COMMA)
+      self.colon = not self.colon
+
+  def write(self):
+    self.doIO(self.protocol.trans.write)
+
+  def read(self):
+    self.doIO(self.protocol.readJSONSyntaxChar)
+
+  def escapeNum(self):
+    return self.colon
+
+  def __str__(self):
+    return '%s, colon=%s' % (self.__class__.__name__, self.colon)
+
+
+class LookaheadReader():
+  hasData = False
+  data = ''
+
+  def __init__(self, protocol):
+    self.protocol = protocol
+
+  def read(self):
+    if self.hasData is True:
+      self.hasData = False
+    else:
+      self.data = self.protocol.trans.read(1)
+    return self.data
+
+  def peek(self):
+    if self.hasData is False:
+      self.data = self.protocol.trans.read(1)
+    self.hasData = True
+    return self.data
+
+class TJSONProtocolBase(TProtocolBase):
+
+  def __init__(self, trans):
+    TProtocolBase.__init__(self, trans)
+    self.resetWriteContext()
+    self.resetReadContext()
+
+  def resetWriteContext(self):
+    self.context = JSONBaseContext(self)
+    self.contextStack = [self.context]
+
+  def resetReadContext(self):
+    self.resetWriteContext()
+    self.reader = LookaheadReader(self)
+
+  def pushContext(self, ctx):
+    self.contextStack.append(ctx)
+    self.context = ctx
+
+  def popContext(self):
+    self.contextStack.pop()
+    if self.contextStack:
+      self.context = self.contextStack[-1]
+    else:
+      self.context = JSONBaseContext(self)
+
+  def writeJSONString(self, string):
+    self.context.write()
+    self.trans.write(json.dumps(string))
+
+  def writeJSONNumber(self, number):
+    self.context.write()
+    jsNumber = str(number)
+    if self.context.escapeNum():
+      jsNumber = "%s%s%s" % (QUOTE, jsNumber,  QUOTE)
+    self.trans.write(jsNumber)
+
+  def writeJSONBase64(self, binary):
+    self.context.write()
+    self.trans.write(QUOTE)
+    self.trans.write(base64.b64encode(binary))
+    self.trans.write(QUOTE)
+
+  def writeJSONObjectStart(self):
+    self.context.write()
+    self.trans.write(LBRACE)
+    self.pushContext(JSONPairContext(self))
+
+  def writeJSONObjectEnd(self):
+    self.popContext()
+    self.trans.write(RBRACE)
+
+  def writeJSONArrayStart(self):
+    self.context.write()
+    self.trans.write(LBRACKET)
+    self.pushContext(JSONListContext(self))
+
+  def writeJSONArrayEnd(self):
+    self.popContext()
+    self.trans.write(RBRACKET)
+
+  def readJSONSyntaxChar(self, character):
+    current = self.reader.read()
+    if character != current:
+      raise TProtocolException(TProtocolException.INVALID_DATA,
+                               "Unexpected character: %s" % current)
+
+  def readJSONString(self, skipContext):
+    string = []
+    if skipContext is False:
+      self.context.read()
+    self.readJSONSyntaxChar(QUOTE)
+    while True:
+      character = self.reader.read()
+      if character == QUOTE:
+        break
+      if character == ESCSEQ[0]:
+        character = self.reader.read()
+        if character == ESCSEQ[1]:
+          self.readJSONSyntaxChar(ZERO)
+          self.readJSONSyntaxChar(ZERO)
+          character = json.JSONDecoder().decode('"\u00%s"' % self.trans.read(2))
+        else:
+          off = ESCAPE_CHAR.find(character)
+          if off == -1:
+            raise TProtocolException(TProtocolException.INVALID_DATA,
+                                     "Expected control char")
+          character = ESCAPE_CHAR_VALS[off]
+      string.append(character)
+    return ''.join(string)
+
+  def isJSONNumeric(self, character):
+    return (True if NUMERIC_CHAR.find(character) != - 1 else False)
+
+  def readJSONQuotes(self):
+    if (self.context.escapeNum()):
+      self.readJSONSyntaxChar(QUOTE)
+
+  def readJSONNumericChars(self):
+    numeric = []
+    while True:
+      character = self.reader.peek()
+      if self.isJSONNumeric(character) is False:
+        break
+      numeric.append(self.reader.read())
+    return ''.join(numeric)
+
+  def readJSONInteger(self):
+    self.context.read()
+    self.readJSONQuotes()
+    numeric = self.readJSONNumericChars()
+    self.readJSONQuotes()
+    try:
+      return int(numeric)
+    except ValueError:
+      raise TProtocolException(TProtocolException.INVALID_DATA,
+                               "Bad data encounted in numeric data")
+
+  def readJSONDouble(self):
+    self.context.read()
+    if self.reader.peek() == QUOTE:
+      string  = self.readJSONString(True)
+      try:
+        double = float(string)
+        if (self.context.escapeNum is False and
+            not math.isinf(double) and
+            not math.isnan(double)):
+          raise TProtocolException(TProtocolException.INVALID_DATA,
+                                   "Numeric data unexpectedly quoted")
+        return double
+      except ValueError:
+        raise TProtocolException(TProtocolException.INVALID_DATA,
+                                 "Bad data encounted in numeric data")
+    else:
+      if self.context.escapeNum() is True:
+        self.readJSONSyntaxChar(QUOTE)
+      try:
+        return float(self.readJSONNumericChars())
+      except ValueError:
+        raise TProtocolException(TProtocolException.INVALID_DATA,
+                                 "Bad data encounted in numeric data")
+
+  def readJSONBase64(self):
+    string = self.readJSONString(False)
+    return base64.b64decode(string)
+
+  def readJSONObjectStart(self):
+    self.context.read()
+    self.readJSONSyntaxChar(LBRACE)
+    self.pushContext(JSONPairContext(self))
+
+  def readJSONObjectEnd(self):
+    self.readJSONSyntaxChar(RBRACE)
+    self.popContext()
+
+  def readJSONArrayStart(self):
+    self.context.read()
+    self.readJSONSyntaxChar(LBRACKET)
+    self.pushContext(JSONListContext(self))
+
+  def readJSONArrayEnd(self):
+    self.readJSONSyntaxChar(RBRACKET)
+    self.popContext()
+
+
+class TJSONProtocol(TJSONProtocolBase):
+
+  def readMessageBegin(self):
+    self.resetReadContext()
+    self.readJSONArrayStart()
+    if self.readJSONInteger() != VERSION:
+      raise TProtocolException(TProtocolException.BAD_VERSION,
+                               "Message contained bad version.")
+    name = self.readJSONString(False)
+    typen = self.readJSONInteger()
+    seqid = self.readJSONInteger()
+    return (name, typen, seqid)
+
+  def readMessageEnd(self):
+    self.readJSONArrayEnd()
+
+  def readStructBegin(self):
+    self.readJSONObjectStart()
+
+  def readStructEnd(self):
+    self.readJSONObjectEnd()
+
+  def readFieldBegin(self):
+    character = self.reader.peek()
+    ttype = 0
+    id = 0
+    if character == RBRACE:
+      ttype = TType.STOP
+    else:
+      id = self.readJSONInteger()
+      self.readJSONObjectStart()
+      ttype = JTYPES[self.readJSONString(False)]
+    return (None, ttype, id)
+
+  def readFieldEnd(self):
+    self.readJSONObjectEnd()
+
+  def readMapBegin(self):
+    self.readJSONArrayStart()
+    keyType = JTYPES[self.readJSONString(False)]
+    valueType = JTYPES[self.readJSONString(False)]
+    size = self.readJSONInteger()
+    self.readJSONObjectStart()
+    return (keyType, valueType, size)
+
+  def readMapEnd(self):
+    self.readJSONObjectEnd()
+    self.readJSONArrayEnd()
+
+  def readCollectionBegin(self):
+    self.readJSONArrayStart()
+    elemType = JTYPES[self.readJSONString(False)]
+    size = self.readJSONInteger()
+    return (elemType, size)
+  readListBegin = readCollectionBegin
+  readSetBegin = readCollectionBegin
+
+  def readCollectionEnd(self):
+    self.readJSONArrayEnd()
+  readSetEnd = readCollectionEnd
+  readListEnd = readCollectionEnd
+
+  def readBool(self):
+    return (False if self.readJSONInteger() == 0 else True)
+
+  def readNumber(self):
+    return self.readJSONInteger()
+  readByte = readNumber
+  readI16 = readNumber
+  readI32 = readNumber
+  readI64 = readNumber
+
+  def readDouble(self):
+    return self.readJSONDouble()
+
+  def readString(self):
+    return self.readJSONString(False)
+
+  def readBinary(self):
+    return self.readJSONBase64()
+
+  def writeMessageBegin(self, name, request_type, seqid):
+    self.resetWriteContext()
+    self.writeJSONArrayStart()
+    self.writeJSONNumber(VERSION)
+    self.writeJSONString(name)
+    self.writeJSONNumber(request_type)
+    self.writeJSONNumber(seqid)
+
+  def writeMessageEnd(self):
+    self.writeJSONArrayEnd()
+
+  def writeStructBegin(self, name):
+    self.writeJSONObjectStart()
+
+  def writeStructEnd(self):
+    self.writeJSONObjectEnd()
+
+  def writeFieldBegin(self, name, ttype, id):
+    self.writeJSONNumber(id)
+    self.writeJSONObjectStart()
+    self.writeJSONString(CTYPES[ttype])
+
+  def writeFieldEnd(self):
+    self.writeJSONObjectEnd()
+
+  def writeFieldStop(self):
+    pass
+
+  def writeMapBegin(self, ktype, vtype, size):
+    self.writeJSONArrayStart()
+    self.writeJSONString(CTYPES[ktype])
+    self.writeJSONString(CTYPES[vtype])
+    self.writeJSONNumber(size)
+    self.writeJSONObjectStart()
+
+  def writeMapEnd(self):
+    self.writeJSONObjectEnd()
+    self.writeJSONArrayEnd()
+    
+  def writeListBegin(self, etype, size):
+    self.writeJSONArrayStart()
+    self.writeJSONString(CTYPES[etype])
+    self.writeJSONNumber(size)
+    
+  def writeListEnd(self):
+    self.writeJSONArrayEnd()
+
+  def writeSetBegin(self, etype, size):
+    self.writeJSONArrayStart()
+    self.writeJSONString(CTYPES[etype])
+    self.writeJSONNumber(size)
+    
+  def writeSetEnd(self):
+    self.writeJSONArrayEnd()
+
+  def writeBool(self, boolean):
+    self.writeJSONNumber(1 if boolean is True else 0)
+
+  def writeInteger(self, integer):
+    self.writeJSONNumber(integer)
+  writeByte = writeInteger
+  writeI16 = writeInteger
+  writeI32 = writeInteger
+  writeI64 = writeInteger
+
+  def writeDouble(self, dbl):
+    self.writeJSONNumber(dbl)
+
+  def writeString(self, string):
+    self.writeJSONString(string)
+    
+  def writeBinary(self, binary):
+    self.writeJSONBase64(binary)
+
+
+class TJSONProtocolFactory:
+
+  def getProtocol(self, trans):
+    return TJSONProtocol(trans)
+
+
+class TSimpleJSONProtocol(TJSONProtocolBase):
+    """Simple, readable, write-only JSON protocol.
+    
+    Useful for interacting with scripting languages.
+    """
+
+    def readMessageBegin(self):
+        raise NotImplementedError()
+    
+    def readMessageEnd(self):
+        raise NotImplementedError()
+    
+    def readStructBegin(self):
+        raise NotImplementedError()
+    
+    def readStructEnd(self):
+        raise NotImplementedError()
+    
+    def writeMessageBegin(self, name, request_type, seqid):
+        self.resetWriteContext()
+    
+    def writeMessageEnd(self):
+        pass
+    
+    def writeStructBegin(self, name):
+        self.writeJSONObjectStart()
+    
+    def writeStructEnd(self):
+        self.writeJSONObjectEnd()
+      
+    def writeFieldBegin(self, name, ttype, fid):
+        self.writeJSONString(name)
+    
+    def writeFieldEnd(self):
+        pass
+    
+    def writeMapBegin(self, ktype, vtype, size):
+        self.writeJSONObjectStart()
+    
+    def writeMapEnd(self):
+        self.writeJSONObjectEnd()
+    
+    def _writeCollectionBegin(self, etype, size):
+        self.writeJSONArrayStart()
+    
+    def _writeCollectionEnd(self):
+        self.writeJSONArrayEnd()
+    writeListBegin = _writeCollectionBegin
+    writeListEnd = _writeCollectionEnd
+    writeSetBegin = _writeCollectionBegin
+    writeSetEnd = _writeCollectionEnd
+
+    def writeInteger(self, integer):
+        self.writeJSONNumber(integer)
+    writeByte = writeInteger
+    writeI16 = writeInteger
+    writeI32 = writeInteger
+    writeI64 = writeInteger
+    
+    def writeBool(self, boolean):
+        self.writeJSONNumber(1 if boolean is True else 0)
+
+    def writeDouble(self, dbl):
+        self.writeJSONNumber(dbl)
+    
+    def writeString(self, string):
+        self.writeJSONString(string)
+      
+    def writeBinary(self, binary):
+        self.writeJSONBase64(binary)
+
+
+class TSimpleJSONProtocolFactory(object):
+
+    def getProtocol(self, trans):
+        return TSimpleJSONProtocol(trans)

+ 406 - 0
desktop/core/ext-py/thrift-0.9.1/src/protocol/TProtocol.py

@@ -0,0 +1,406 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from thrift.Thrift import *
+
+
+class TProtocolException(TException):
+  """Custom Protocol Exception class"""
+
+  UNKNOWN = 0
+  INVALID_DATA = 1
+  NEGATIVE_SIZE = 2
+  SIZE_LIMIT = 3
+  BAD_VERSION = 4
+
+  def __init__(self, type=UNKNOWN, message=None):
+    TException.__init__(self, message)
+    self.type = type
+
+
+class TProtocolBase:
+  """Base class for Thrift protocol driver."""
+
+  def __init__(self, trans):
+    self.trans = trans
+
+  def writeMessageBegin(self, name, ttype, seqid):
+    pass
+
+  def writeMessageEnd(self):
+    pass
+
+  def writeStructBegin(self, name):
+    pass
+
+  def writeStructEnd(self):
+    pass
+
+  def writeFieldBegin(self, name, ttype, fid):
+    pass
+
+  def writeFieldEnd(self):
+    pass
+
+  def writeFieldStop(self):
+    pass
+
+  def writeMapBegin(self, ktype, vtype, size):
+    pass
+
+  def writeMapEnd(self):
+    pass
+
+  def writeListBegin(self, etype, size):
+    pass
+
+  def writeListEnd(self):
+    pass
+
+  def writeSetBegin(self, etype, size):
+    pass
+
+  def writeSetEnd(self):
+    pass
+
+  def writeBool(self, bool_val):
+    pass
+
+  def writeByte(self, byte):
+    pass
+
+  def writeI16(self, i16):
+    pass
+
+  def writeI32(self, i32):
+    pass
+
+  def writeI64(self, i64):
+    pass
+
+  def writeDouble(self, dub):
+    pass
+
+  def writeString(self, str_val):
+    pass
+
+  def readMessageBegin(self):
+    pass
+
+  def readMessageEnd(self):
+    pass
+
+  def readStructBegin(self):
+    pass
+
+  def readStructEnd(self):
+    pass
+
+  def readFieldBegin(self):
+    pass
+
+  def readFieldEnd(self):
+    pass
+
+  def readMapBegin(self):
+    pass
+
+  def readMapEnd(self):
+    pass
+
+  def readListBegin(self):
+    pass
+
+  def readListEnd(self):
+    pass
+
+  def readSetBegin(self):
+    pass
+
+  def readSetEnd(self):
+    pass
+
+  def readBool(self):
+    pass
+
+  def readByte(self):
+    pass
+
+  def readI16(self):
+    pass
+
+  def readI32(self):
+    pass
+
+  def readI64(self):
+    pass
+
+  def readDouble(self):
+    pass
+
+  def readString(self):
+    pass
+
+  def skip(self, ttype):
+    if ttype == TType.STOP:
+      return
+    elif ttype == TType.BOOL:
+      self.readBool()
+    elif ttype == TType.BYTE:
+      self.readByte()
+    elif ttype == TType.I16:
+      self.readI16()
+    elif ttype == TType.I32:
+      self.readI32()
+    elif ttype == TType.I64:
+      self.readI64()
+    elif ttype == TType.DOUBLE:
+      self.readDouble()
+    elif ttype == TType.STRING:
+      self.readString()
+    elif ttype == TType.STRUCT:
+      name = self.readStructBegin()
+      while True:
+        (name, ttype, id) = self.readFieldBegin()
+        if ttype == TType.STOP:
+          break
+        self.skip(ttype)
+        self.readFieldEnd()
+      self.readStructEnd()
+    elif ttype == TType.MAP:
+      (ktype, vtype, size) = self.readMapBegin()
+      for i in xrange(size):
+        self.skip(ktype)
+        self.skip(vtype)
+      self.readMapEnd()
+    elif ttype == TType.SET:
+      (etype, size) = self.readSetBegin()
+      for i in xrange(size):
+        self.skip(etype)
+      self.readSetEnd()
+    elif ttype == TType.LIST:
+      (etype, size) = self.readListBegin()
+      for i in xrange(size):
+        self.skip(etype)
+      self.readListEnd()
+
+  # tuple of: ( 'reader method' name, is_container bool, 'writer_method' name )
+  _TTYPE_HANDLERS = (
+       (None, None, False),  # 0 TType.STOP
+       (None, None, False),  # 1 TType.VOID # TODO: handle void?
+       ('readBool', 'writeBool', False),  # 2 TType.BOOL
+       ('readByte',  'writeByte', False),  # 3 TType.BYTE and I08
+       ('readDouble', 'writeDouble', False),  # 4 TType.DOUBLE
+       (None, None, False),  # 5 undefined
+       ('readI16', 'writeI16', False),  # 6 TType.I16
+       (None, None, False),  # 7 undefined
+       ('readI32', 'writeI32', False),  # 8 TType.I32
+       (None, None, False),  # 9 undefined
+       ('readI64', 'writeI64', False),  # 10 TType.I64
+       ('readString', 'writeString', False),  # 11 TType.STRING and UTF7
+       ('readContainerStruct', 'writeContainerStruct', True),  # 12 *.STRUCT
+       ('readContainerMap', 'writeContainerMap', True),  # 13 TType.MAP
+       ('readContainerSet', 'writeContainerSet', True),  # 14 TType.SET
+       ('readContainerList', 'writeContainerList', True),  # 15 TType.LIST
+       (None, None, False),  # 16 TType.UTF8 # TODO: handle utf8 types?
+       (None, None, False)  # 17 TType.UTF16 # TODO: handle utf16 types?
+      )
+
+  def readFieldByTType(self, ttype, spec):
+    try:
+      (r_handler, w_handler, is_container) = self._TTYPE_HANDLERS[ttype]
+    except IndexError:
+      raise TProtocolException(type=TProtocolException.INVALID_DATA,
+                               message='Invalid field type %d' % (ttype))
+    if r_handler is None:
+      raise TProtocolException(type=TProtocolException.INVALID_DATA,
+                               message='Invalid field type %d' % (ttype))
+    reader = getattr(self, r_handler)
+    if not is_container:
+      return reader()
+    return reader(spec)
+
+  def readContainerList(self, spec):
+    results = []
+    ttype, tspec = spec[0], spec[1]
+    r_handler = self._TTYPE_HANDLERS[ttype][0]
+    reader = getattr(self, r_handler)
+    (list_type, list_len) = self.readListBegin()
+    if tspec is None:
+      # list values are simple types
+      for idx in xrange(list_len):
+        results.append(reader())
+    else:
+      # this is like an inlined readFieldByTType
+      container_reader = self._TTYPE_HANDLERS[list_type][0]
+      val_reader = getattr(self, container_reader)
+      for idx in xrange(list_len):
+        val = val_reader(tspec)
+        results.append(val)
+    self.readListEnd()
+    return results
+
+  def readContainerSet(self, spec):
+    results = set()
+    ttype, tspec = spec[0], spec[1]
+    r_handler = self._TTYPE_HANDLERS[ttype][0]
+    reader = getattr(self, r_handler)
+    (set_type, set_len) = self.readSetBegin()
+    if tspec is None:
+      # set members are simple types
+      for idx in xrange(set_len):
+        results.add(reader())
+    else:
+      container_reader = self._TTYPE_HANDLERS[set_type][0]
+      val_reader = getattr(self, container_reader)
+      for idx in xrange(set_len):
+        results.add(val_reader(tspec))
+    self.readSetEnd()
+    return results
+
+  def readContainerStruct(self, spec):
+    (obj_class, obj_spec) = spec
+    obj = obj_class()
+    obj.read(self)
+    return obj
+
+  def readContainerMap(self, spec):
+    results = dict()
+    key_ttype, key_spec = spec[0], spec[1]
+    val_ttype, val_spec = spec[2], spec[3]
+    (map_ktype, map_vtype, map_len) = self.readMapBegin()
+    # TODO: compare types we just decoded with thrift_spec and
+    # abort/skip if types disagree
+    key_reader = getattr(self, self._TTYPE_HANDLERS[key_ttype][0])
+    val_reader = getattr(self, self._TTYPE_HANDLERS[val_ttype][0])
+    # list values are simple types
+    for idx in xrange(map_len):
+      if key_spec is None:
+        k_val = key_reader()
+      else:
+        k_val = self.readFieldByTType(key_ttype, key_spec)
+      if val_spec is None:
+        v_val = val_reader()
+      else:
+        v_val = self.readFieldByTType(val_ttype, val_spec)
+      # this raises a TypeError with unhashable keys types
+      # i.e. this fails: d=dict(); d[[0,1]] = 2
+      results[k_val] = v_val
+    self.readMapEnd()
+    return results
+
+  def readStruct(self, obj, thrift_spec):
+    self.readStructBegin()
+    while True:
+      (fname, ftype, fid) = self.readFieldBegin()
+      if ftype == TType.STOP:
+        break
+      try:
+        field = thrift_spec[fid]
+      except IndexError:
+        self.skip(ftype)
+      else:
+        if field is not None and ftype == field[1]:
+          fname = field[2]
+          fspec = field[3]
+          val = self.readFieldByTType(ftype, fspec)
+          setattr(obj, fname, val)
+        else:
+          self.skip(ftype)
+      self.readFieldEnd()
+    self.readStructEnd()
+
+  def writeContainerStruct(self, val, spec):
+    val.write(self)
+
+  def writeContainerList(self, val, spec):
+    self.writeListBegin(spec[0], len(val))
+    r_handler, w_handler, is_container = self._TTYPE_HANDLERS[spec[0]]
+    e_writer = getattr(self, w_handler)
+    if not is_container:
+      for elem in val:
+        e_writer(elem)
+    else:
+      for elem in val:
+        e_writer(elem, spec[1])
+    self.writeListEnd()
+
+  def writeContainerSet(self, val, spec):
+    self.writeSetBegin(spec[0], len(val))
+    r_handler, w_handler, is_container = self._TTYPE_HANDLERS[spec[0]]
+    e_writer = getattr(self, w_handler)
+    if not is_container:
+      for elem in val:
+        e_writer(elem)
+    else:
+      for elem in val:
+        e_writer(elem, spec[1])
+    self.writeSetEnd()
+
+  def writeContainerMap(self, val, spec):
+    k_type = spec[0]
+    v_type = spec[2]
+    ignore, ktype_name, k_is_container = self._TTYPE_HANDLERS[k_type]
+    ignore, vtype_name, v_is_container = self._TTYPE_HANDLERS[v_type]
+    k_writer = getattr(self, ktype_name)
+    v_writer = getattr(self, vtype_name)
+    self.writeMapBegin(k_type, v_type, len(val))
+    for m_key, m_val in val.iteritems():
+      if not k_is_container:
+        k_writer(m_key)
+      else:
+        k_writer(m_key, spec[1])
+      if not v_is_container:
+        v_writer(m_val)
+      else:
+        v_writer(m_val, spec[3])
+    self.writeMapEnd()
+
+  def writeStruct(self, obj, thrift_spec):
+    self.writeStructBegin(obj.__class__.__name__)
+    for field in thrift_spec:
+      if field is None:
+        continue
+      fname = field[2]
+      val = getattr(obj, fname)
+      if val is None:
+        # skip writing out unset fields
+        continue
+      fid = field[0]
+      ftype = field[1]
+      fspec = field[3]
+      # get the writer method for this value
+      self.writeFieldBegin(fname, ftype, fid)
+      self.writeFieldByTType(ftype, val, fspec)
+      self.writeFieldEnd()
+    self.writeFieldStop()
+    self.writeStructEnd()
+
+  def writeFieldByTType(self, ttype, val, spec):
+    r_handler, w_handler, is_container = self._TTYPE_HANDLERS[ttype]
+    writer = getattr(self, w_handler)
+    if is_container:
+      writer(val, spec)
+    else:
+      writer(val)
+
+
+class TProtocolFactory:
+  def getProtocol(self, trans):
+    pass

+ 1 - 1
desktop/core/ext-py/thrift/src/protocol/__init__.py → desktop/core/ext-py/thrift-0.9.1/src/protocol/__init__.py

@@ -17,4 +17,4 @@
 # under the License.
 #
 
-__all__ = ['TProtocol', 'TBinaryProtocol', 'fastbinary']
+__all__ = ['fastbinary', 'TBase', 'TBinaryProtocol', 'TCompactProtocol', 'TJSONProtocol', 'TProtocol']

+ 21 - 5
desktop/core/ext-py/thrift/src/protocol/fastbinary.c → desktop/core/ext-py/thrift-0.9.1/src/protocol/fastbinary.c

@@ -19,9 +19,24 @@
 
 #include <Python.h>
 #include "cStringIO.h"
-#include <stdbool.h>
 #include <stdint.h>
-#include <netinet/in.h>
+#ifndef _WIN32
+# include <stdbool.h>
+# include <netinet/in.h>
+#else
+# include <WinSock2.h>
+# pragma comment (lib, "ws2_32.lib")
+# define BIG_ENDIAN (4321)
+# define LITTLE_ENDIAN (1234)
+# define BYTE_ORDER LITTLE_ENDIAN
+# if defined(_MSC_VER) && _MSC_VER < 1600
+   typedef int _Bool;
+#  define bool _Bool
+#  define false 0 
+#  define true 1
+# endif
+# define inline __inline
+#endif
 
 /* Fix endianness issues on Solaris */
 #if defined (__SVR4) && defined (__sun)
@@ -1110,11 +1125,12 @@ decode_val(DecodeBuffer* input, TType type, PyObject* typeargs) {
 
   case T_STRUCT: {
     StructTypeArgs parsedargs;
+	PyObject* ret;
     if (!parse_struct_args(&parsedargs, typeargs)) {
       return NULL;
     }
 
-    PyObject* ret = PyObject_CallObject(parsedargs.klass, NULL);
+    ret = PyObject_CallObject(parsedargs.klass, NULL);
     if (!ret) {
       return NULL;
     }
@@ -1147,8 +1163,8 @@ decode_binary(PyObject *self, PyObject *args) {
   PyObject* transport = NULL;
   PyObject* typeargs = NULL;
   StructTypeArgs parsedargs;
-  DecodeBuffer input = {};
-
+  DecodeBuffer input = {0, 0};
+  
   if (!PyArg_ParseTuple(args, "OOO", &output_obj, &transport, &typeargs)) {
     return NULL;
   }

+ 37 - 13
desktop/core/ext-py/thrift/src/server/THttpServer.py → desktop/core/ext-py/thrift-0.9.1/src/server/THttpServer.py

@@ -22,19 +22,37 @@ import BaseHTTPServer
 from thrift.server import TServer
 from thrift.transport import TTransport
 
+
+class ResponseException(Exception):
+  """Allows handlers to override the HTTP response
+
+  Normally, THttpServer always sends a 200 response.  If a handler wants
+  to override this behavior (e.g., to simulate a misconfigured or
+  overloaded web server during testing), it can raise a ResponseException.
+  The function passed to the constructor will be called with the
+  RequestHandler as its only argument.
+  """
+  def __init__(self, handler):
+    self.handler = handler
+
+
 class THttpServer(TServer.TServer):
   """A simple HTTP-based Thrift server
 
   This class is not very performant, but it is useful (for example) for
-  acting as a mock version of an Apache-based PHP Thrift endpoint."""
-
-  def __init__(self, processor, server_address,
-      inputProtocolFactory, outputProtocolFactory = None):
+  acting as a mock version of an Apache-based PHP Thrift endpoint.
+  """
+  def __init__(self,
+               processor,
+               server_address,
+               inputProtocolFactory,
+               outputProtocolFactory=None,
+               server_class=BaseHTTPServer.HTTPServer):
     """Set up protocol factories and HTTP server.
 
     See BaseHTTPServer for server_address.
-    See TServer for protocol factories."""
-
+    See TServer for protocol factories.
+    """
     if outputProtocolFactory is None:
       outputProtocolFactory = inputProtocolFactory
 
@@ -46,18 +64,24 @@ class THttpServer(TServer.TServer):
     class RequestHander(BaseHTTPServer.BaseHTTPRequestHandler):
       def do_POST(self):
         # Don't care about the request path.
-        self.send_response(200)
-        self.send_header("content-type", "application/x-thrift")
-        self.end_headers()
-
         itrans = TTransport.TFileObjectTransport(self.rfile)
         otrans = TTransport.TFileObjectTransport(self.wfile)
+        itrans = TTransport.TBufferedTransport(
+          itrans, int(self.headers['Content-Length']))
+        otrans = TTransport.TMemoryBuffer()
         iprot = thttpserver.inputProtocolFactory.getProtocol(itrans)
         oprot = thttpserver.outputProtocolFactory.getProtocol(otrans)
-        thttpserver.processor.process(iprot, oprot)
-        otrans.flush()
+        try:
+          thttpserver.processor.process(iprot, oprot)
+        except ResponseException, exn:
+          exn.handler(self)
+        else:
+          self.send_response(200)
+          self.send_header("content-type", "application/x-thrift")
+          self.end_headers()
+          self.wfile.write(otrans.getvalue())
 
-    self.httpd = BaseHTTPServer.HTTPServer(server_address, RequestHander)
+    self.httpd = server_class(server_address, RequestHander)
 
   def serve(self):
     self.httpd.serve_forever()

+ 80 - 43
desktop/core/ext-py/thrift/src/server/TNonblockingServer.py → desktop/core/ext-py/thrift-0.9.1/src/server/TNonblockingServer.py

@@ -18,10 +18,11 @@
 #
 """Implementation of non-blocking server.
 
-The main idea of the server is reciving and sending requests
-only from main thread.
+The main idea of the server is to receive and send requests
+only from the main thread.
 
-It also makes thread pool server in tasks terms, not connections.
+The thread poool should be sized for concurrent tasks, not
+maximum connections
 """
 import threading
 import socket
@@ -35,8 +36,10 @@ from thrift.protocol.TBinaryProtocol import TBinaryProtocolFactory
 
 __all__ = ['TNonblockingServer']
 
+
 class Worker(threading.Thread):
     """Worker is a small helper to process incoming connection."""
+
     def __init__(self, queue):
         threading.Thread.__init__(self)
         self.queue = queue
@@ -60,8 +63,9 @@ WAIT_PROCESS = 2
 SEND_ANSWER = 3
 CLOSED = 4
 
+
 def locked(func):
-    "Decorator which locks self.lock."
+    """Decorator which locks self.lock."""
     def nested(self, *args, **kwargs):
         self.lock.acquire()
         try:
@@ -70,8 +74,9 @@ def locked(func):
             self.lock.release()
     return nested
 
+
 def socket_exception(func):
-    "Decorator close object on socket.error."
+    """Decorator close object on socket.error."""
     def read(self, *args, **kwargs):
         try:
             return func(self, *args, **kwargs)
@@ -79,16 +84,17 @@ def socket_exception(func):
             self.close()
     return read
 
+
 class Connection:
     """Basic class is represented connection.
-    
+
     It can be in state:
         WAIT_LEN --- connection is reading request len.
         WAIT_MESSAGE --- connection is reading request.
-        WAIT_PROCESS --- connection has just read whole request and 
-            waits for call ready routine.
+        WAIT_PROCESS --- connection has just read whole request and
+                         waits for call ready routine.
         SEND_ANSWER --- connection is sending answer string (including length
-            of answer).
+                        of answer).
         CLOSED --- socket was closed and connection should be deleted.
     """
     def __init__(self, new_socket, wake_up):
@@ -102,13 +108,13 @@ class Connection:
 
     def _read_len(self):
         """Reads length of request.
-        
-        It's really paranoic routine and it may be replaced by 
-        self.socket.recv(4)."""
+
+        It's a safer alternative to self.socket.recv(4)
+        """
         read = self.socket.recv(4 - len(self.message))
         if len(read) == 0:
-            # if we read 0 bytes and self.message is empty, it means client close 
-            # connection
+            # if we read 0 bytes and self.message is empty, then
+            # the client closed the connection
             if len(self.message) != 0:
                 logging.error("can't read frame size from socket")
             self.close()
@@ -117,8 +123,8 @@ class Connection:
         if len(self.message) == 4:
             self.len, = struct.unpack('!i', self.message)
             if self.len < 0:
-                logging.error("negative frame size, it seems client"\
-                    " doesn't use FramedTransport")
+                logging.error("negative frame size, it seems client "
+                              "doesn't use FramedTransport")
                 self.close()
             elif self.len == 0:
                 logging.error("empty frame, it's really strange")
@@ -139,8 +145,8 @@ class Connection:
         elif self.status == WAIT_MESSAGE:
             read = self.socket.recv(self.len - len(self.message))
             if len(read) == 0:
-                logging.error("can't read frame from socket (get %d of %d bytes)" %
-                    (len(self.message), self.len))
+                logging.error("can't read frame from socket (get %d of "
+                              "%d bytes)" % (len(self.message), self.len))
                 self.close()
                 return
             self.message += read
@@ -162,14 +168,14 @@ class Connection:
     @locked
     def ready(self, all_ok, message):
         """Callback function for switching state and waking up main thread.
-        
+
         This function is the only function witch can be called asynchronous.
-        
+
         The ready can switch Connection to three states:
             WAIT_LEN if request was oneway.
             SEND_ANSWER if request was processed in normal way.
             CLOSED if request throws unexpected exception.
-        
+
         The one wakes up main thread.
         """
         assert self.status == WAIT_PROCESS
@@ -178,43 +184,50 @@ class Connection:
             self.wake_up()
             return
         self.len = ''
-        self.message = struct.pack('!i', len(message)) + message
         if len(message) == 0:
             # it was a oneway request, do not write answer
+            self.message = ''
             self.status = WAIT_LEN
         else:
+            self.message = struct.pack('!i', len(message)) + message
             self.status = SEND_ANSWER
         self.wake_up()
 
     @locked
     def is_writeable(self):
-        "Returns True if connection should be added to write list of select."
+        """Return True if connection should be added to write list of select"""
         return self.status == SEND_ANSWER
 
     # it's not necessary, but...
     @locked
     def is_readable(self):
-        "Returns True if connection should be added to read list of select."
+        """Return True if connection should be added to read list of select"""
         return self.status in (WAIT_LEN, WAIT_MESSAGE)
 
     @locked
     def is_closed(self):
-        "Returns True if connection is closed."
+        """Returns True if connection is closed."""
         return self.status == CLOSED
 
     def fileno(self):
-        "Returns the file descriptor of the associated socket."
+        """Returns the file descriptor of the associated socket."""
         return self.socket.fileno()
 
     def close(self):
-        "Closes connection"
+        """Closes connection"""
         self.status = CLOSED
         self.socket.close()
 
+
 class TNonblockingServer:
     """Non-blocking server."""
-    def __init__(self, processor, lsocket, inputProtocolFactory=None, 
-            outputProtocolFactory=None, threads=10):
+
+    def __init__(self,
+                 processor,
+                 lsocket,
+                 inputProtocolFactory=None,
+                 outputProtocolFactory=None,
+                 threads=10):
         self.processor = processor
         self.socket = lsocket
         self.in_protocol = inputProtocolFactory or TBinaryProtocolFactory()
@@ -224,15 +237,18 @@ class TNonblockingServer:
         self.tasks = Queue.Queue()
         self._read, self._write = socket.socketpair()
         self.prepared = False
+        self._stop = False
 
     def setNumThreads(self, num):
         """Set the number of worker threads that should be created."""
         # implement ThreadPool interface
-        assert not self.prepared, "You can't change number of threads for working server"
+        assert not self.prepared, "Can't change number of threads after start"
         self.threads = num
 
     def prepare(self):
         """Prepares server for serve requests."""
+        if self.prepared:
+            return
         self.socket.listen()
         for _ in xrange(self.threads):
             thread = Worker(self.tasks)
@@ -242,16 +258,32 @@ class TNonblockingServer:
 
     def wake_up(self):
         """Wake up main thread.
-        
+
         The server usualy waits in select call in we should terminate one.
         The simplest way is using socketpair.
-        
+
         Select always wait to read from the first socket of socketpair.
-        
+
         In this case, we can just write anything to the second socket from
-        socketpair."""
+        socketpair.
+        """
         self._write.send('1')
 
+    def stop(self):
+        """Stop the server.
+
+        This method causes the serve() method to return.  stop() may be invoked
+        from within your handler, or from another thread.
+
+        After stop() is called, serve() will return but the server will still
+        be listening on the socket.  serve() may then be called again to resume
+        processing requests.  Alternatively, close() may be called after
+        serve() returns to close the server socket and shutdown all worker
+        threads.
+        """
+        self._stop = True
+        self.wake_up()
+
     def _select(self):
         """Does select on open connections."""
         readable = [self.socket.handle.fileno(), self._read.fileno()]
@@ -264,21 +296,22 @@ class TNonblockingServer:
             if connection.is_closed():
                 del self.clients[i]
         return select.select(readable, writable, readable)
-        
+
     def handle(self):
         """Handle requests.
-       
-        WARNING! You must call prepare BEFORE calling handle.
+
+        WARNING! You must call prepare() BEFORE calling handle()
         """
         assert self.prepared, "You have to call prepare before handle"
         rset, wset, xset = self._select()
         for readable in rset:
             if readable == self._read.fileno():
                 # don't care i just need to clean readable flag
-                self._read.recv(1024) 
+                self._read.recv(1024)
             elif readable == self.socket.handle.fileno():
                 client = self.socket.accept().handle
-                self.clients[client.fileno()] = Connection(client, self.wake_up)
+                self.clients[client.fileno()] = Connection(client,
+                                                           self.wake_up)
             else:
                 connection = self.clients[readable]
                 connection.read()
@@ -287,7 +320,7 @@ class TNonblockingServer:
                     otransport = TTransport.TMemoryBuffer()
                     iprot = self.in_protocol.getProtocol(itransport)
                     oprot = self.out_protocol.getProtocol(otransport)
-                    self.tasks.put([self.processor, iprot, oprot, 
+                    self.tasks.put([self.processor, iprot, oprot,
                                     otransport, connection.ready])
         for writeable in wset:
             self.clients[writeable].write()
@@ -301,9 +334,13 @@ class TNonblockingServer:
             self.tasks.put([None, None, None, None, None])
         self.socket.close()
         self.prepared = False
-        
+
     def serve(self):
-        """Serve forever."""
+        """Serve requests.
+
+        Serve requests forever, or until stop() is called.
+        """
+        self._stop = False
         self.prepare()
-        while True:
+        while not self._stop:
             self.handle()

+ 118 - 0
desktop/core/ext-py/thrift-0.9.1/src/server/TProcessPoolServer.py

@@ -0,0 +1,118 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+import logging
+from multiprocessing import  Process, Value, Condition, reduction
+
+from TServer import TServer
+from thrift.transport.TTransport import TTransportException
+
+
+class TProcessPoolServer(TServer):
+    """Server with a fixed size pool of worker subprocesses to service requests
+
+    Note that if you need shared state between the handlers - it's up to you!
+    Written by Dvir Volk, doat.com
+    """
+    def __init__(self, *args):
+        TServer.__init__(self, *args)
+        self.numWorkers = 10
+        self.workers = []
+        self.isRunning = Value('b', False)
+        self.stopCondition = Condition()
+        self.postForkCallback = None
+
+    def setPostForkCallback(self, callback):
+        if not callable(callback):
+            raise TypeError("This is not a callback!")
+        self.postForkCallback = callback
+
+    def setNumWorkers(self, num):
+        """Set the number of worker threads that should be created"""
+        self.numWorkers = num
+
+    def workerProcess(self):
+        """Loop getting clients from the shared queue and process them"""
+        if self.postForkCallback:
+            self.postForkCallback()
+
+        while self.isRunning.value:
+            try:
+                client = self.serverTransport.accept()
+                self.serveClient(client)
+            except (KeyboardInterrupt, SystemExit):
+                return 0
+            except Exception, x:
+                logging.exception(x)
+
+    def serveClient(self, client):
+        """Process input/output from a client for as long as possible"""
+        itrans = self.inputTransportFactory.getTransport(client)
+        otrans = self.outputTransportFactory.getTransport(client)
+        iprot = self.inputProtocolFactory.getProtocol(itrans)
+        oprot = self.outputProtocolFactory.getProtocol(otrans)
+
+        try:
+            while True:
+                self.processor.process(iprot, oprot)
+        except TTransportException, tx:
+            pass
+        except Exception, x:
+            logging.exception(x)
+
+        itrans.close()
+        otrans.close()
+
+    def serve(self):
+        """Start workers and put into queue"""
+        # this is a shared state that can tell the workers to exit when False
+        self.isRunning.value = True
+
+        # first bind and listen to the port
+        self.serverTransport.listen()
+
+        # fork the children
+        for i in range(self.numWorkers):
+            try:
+                w = Process(target=self.workerProcess)
+                w.daemon = True
+                w.start()
+                self.workers.append(w)
+            except Exception, x:
+                logging.exception(x)
+
+        # wait until the condition is set by stop()
+        while True:
+            self.stopCondition.acquire()
+            try:
+                self.stopCondition.wait()
+                break
+            except (SystemExit, KeyboardInterrupt):
+                break
+            except Exception, x:
+                logging.exception(x)
+
+        self.isRunning.value = False
+
+    def stop(self):
+        self.isRunning.value = False
+        self.stopCondition.acquire()
+        self.stopCondition.notify()
+        self.stopCondition.release()

+ 22 - 23
desktop/core/ext-py/thrift/src/server/TServer.py → desktop/core/ext-py/thrift-0.9.1/src/server/TServer.py

@@ -17,27 +17,28 @@
 # under the License.
 #
 
+import Queue
 import logging
-import sys
 import os
-import traceback
+import sys
 import threading
-import Queue
+import traceback
 
 from thrift.Thrift import TProcessor
-from thrift.transport import TTransport
 from thrift.protocol import TBinaryProtocol
+from thrift.transport import TTransport
 
-class TServer:
 
-  """Base interface for a server, which must have a serve method."""
+class TServer:
+  """Base interface for a server, which must have a serve() method.
 
-  """ 3 constructors for all servers:
+  Three constructors for all servers:
   1) (processor, serverTransport)
   2) (processor, serverTransport, transportFactory, protocolFactory)
   3) (processor, serverTransport,
       inputTransportFactory, outputTransportFactory,
-      inputProtocolFactory, outputProtocolFactory)"""
+      inputProtocolFactory, outputProtocolFactory)
+  """
   def __init__(self, *args):
     if (len(args) == 2):
       self.__initArgs__(args[0], args[1],
@@ -63,8 +64,8 @@ class TServer:
   def serve(self):
     pass
 
-class TSimpleServer(TServer):
 
+class TSimpleServer(TServer):
   """Simple single-threaded server that just pumps around one transport."""
 
   def __init__(self, *args):
@@ -89,19 +90,21 @@ class TSimpleServer(TServer):
       itrans.close()
       otrans.close()
 
-class TThreadedServer(TServer):
 
+class TThreadedServer(TServer):
   """Threaded server that spawns a new thread per each connection."""
 
-  def __init__(self, *args):
+  def __init__(self, *args, **kwargs):
     TServer.__init__(self, *args)
+    self.daemon = kwargs.get("daemon", False)
 
   def serve(self):
     self.serverTransport.listen()
     while True:
       try:
         client = self.serverTransport.accept()
-        t = threading.Thread(target = self.handle, args=(client,))
+        t = threading.Thread(target=self.handle, args=(client,))
+        t.setDaemon(self.daemon)
         t.start()
       except KeyboardInterrupt:
         raise
@@ -124,14 +127,15 @@ class TThreadedServer(TServer):
     itrans.close()
     otrans.close()
 
-class TThreadPoolServer(TServer):
 
+class TThreadPoolServer(TServer):
   """Server with a fixed size pool of threads which service requests."""
 
-  def __init__(self, *args):
+  def __init__(self, *args, **kwargs):
     TServer.__init__(self, *args)
     self.clients = Queue.Queue()
     self.threads = 10
+    self.daemon = kwargs.get("daemon", False)
 
   def setNumThreads(self, num):
     """Set the number of worker threads that should be created"""
@@ -167,7 +171,8 @@ class TThreadPoolServer(TServer):
     """Start a fixed number of worker threads and put client into a queue"""
     for i in range(self.threads):
       try:
-        t = threading.Thread(target = self.serveThread)
+        t = threading.Thread(target=self.serveThread)
+        t.setDaemon(self.daemon)
         t.start()
       except Exception, x:
         logging.exception(x)
@@ -183,9 +188,8 @@ class TThreadPoolServer(TServer):
 
 
 class TForkingServer(TServer):
+  """A Thrift server that forks a new process for each request
 
-  """A Thrift server that forks a new process for each request"""
-  """
   This is more scalable than the threaded server as it does not cause
   GIL contention.
 
@@ -196,7 +200,6 @@ class TForkingServer(TServer):
   This code is heavily inspired by SocketServer.ForkingMixIn in the
   Python stdlib.
   """
-
   def __init__(self, *args):
     TServer.__init__(self, *args)
     self.children = []
@@ -208,14 +211,13 @@ class TForkingServer(TServer):
       except IOError, e:
         logging.warning(e, exc_info=True)
 
-
     self.serverTransport.listen()
     while True:
       client = self.serverTransport.accept()
       try:
         pid = os.fork()
 
-        if pid: # parent
+        if pid:  # parent
           # add before collect, otherwise you race w/ waitpid
           self.children.append(pid)
           self.collect_children()
@@ -254,7 +256,6 @@ class TForkingServer(TServer):
       except Exception, x:
         logging.exception(x)
 
-
   def collect_children(self):
     while self.children:
       try:
@@ -266,5 +267,3 @@ class TForkingServer(TServer):
         self.children.remove(pid)
       else:
         break
-
-

+ 0 - 0
desktop/core/ext-py/thrift/src/server/__init__.py → desktop/core/ext-py/thrift-0.9.1/src/server/__init__.py


+ 59 - 10
desktop/core/ext-py/thrift/src/transport/THttpClient.py → desktop/core/ext-py/thrift-0.9.1/src/transport/THttpClient.py

@@ -17,15 +17,20 @@
 # under the License.
 #
 
-from TTransport import *
-from cStringIO import StringIO
-
-import urlparse
 import httplib
+import os
+import socket
+import sys
+import urllib
+import urlparse
 import warnings
 
-class THttpClient(TTransportBase):
+from cStringIO import StringIO
+
+from TTransport import *
 
+
+class THttpClient(TTransportBase):
   """Http implementation of TTransport base."""
 
   def __init__(self, uri_or_host, port=None, path=None):
@@ -34,10 +39,13 @@ class THttpClient(TTransportBase):
     THttpClient(host, port, path) - deprecated
     THttpClient(uri)
 
-    Only the second supports https."""
-
+    Only the second supports https.
+    """
     if port is not None:
-      warnings.warn("Please use the THttpClient('http://host:port/path') syntax", DeprecationWarning, stacklevel=2)
+      warnings.warn(
+        "Please use the THttpClient('http://host:port/path') syntax",
+        DeprecationWarning,
+        stacklevel=2)
       self.host = uri_or_host
       self.port = port
       assert path
@@ -53,8 +61,12 @@ class THttpClient(TTransportBase):
         self.port = parsed.port or httplib.HTTPS_PORT
       self.host = parsed.hostname
       self.path = parsed.path
+      if parsed.query:
+        self.path += '?%s' % parsed.query
     self.__wbuf = StringIO()
     self.__http = None
+    self.__timeout = None
+    self.__custom_headers = None
 
   def open(self):
     if self.scheme == 'http':
@@ -67,7 +79,19 @@ class THttpClient(TTransportBase):
     self.__http = None
 
   def isOpen(self):
-    return self.__http != None
+    return self.__http is not None
+
+  def setTimeout(self, ms):
+    if not hasattr(socket, 'getdefaulttimeout'):
+      raise NotImplementedError
+
+    if ms is None:
+      self.__timeout = None
+    else:
+      self.__timeout = ms / 1000.0
+
+  def setCustomHeaders(self, headers):
+    self.__custom_headers = headers
 
   def read(self, sz):
     return self.__http.file.read(sz)
@@ -75,10 +99,19 @@ class THttpClient(TTransportBase):
   def write(self, buf):
     self.__wbuf.write(buf)
 
+  def __withTimeout(f):
+    def _f(*args, **kwargs):
+      orig_timeout = socket.getdefaulttimeout()
+      socket.setdefaulttimeout(args[0].__timeout)
+      result = f(*args, **kwargs)
+      socket.setdefaulttimeout(orig_timeout)
+      return result
+    return _f
+
   def flush(self):
     if self.isOpen():
       self.close()
-    self.open();
+    self.open()
 
     # Pull data out of buffer
     data = self.__wbuf.getvalue()
@@ -91,6 +124,18 @@ class THttpClient(TTransportBase):
     self.__http.putheader('Host', self.host)
     self.__http.putheader('Content-Type', 'application/x-thrift')
     self.__http.putheader('Content-Length', str(len(data)))
+
+    if not self.__custom_headers or 'User-Agent' not in self.__custom_headers:
+      user_agent = 'Python/THttpClient'
+      script = os.path.basename(sys.argv[0])
+      if script:
+        user_agent = '%s (%s)' % (user_agent, urllib.quote(script))
+      self.__http.putheader('User-Agent', user_agent)
+
+    if self.__custom_headers:
+        for key, val in self.__custom_headers.iteritems():
+            self.__http.putheader(key, val)
+
     self.__http.endheaders()
 
     # Write payload
@@ -98,3 +143,7 @@ class THttpClient(TTransportBase):
 
     # Get reply to flush the request
     self.code, self.message, self.headers = self.__http.getreply()
+
+  # Decorate if we know how to timeout
+  if hasattr(socket, 'getdefaulttimeout'):
+    flush = __withTimeout(flush)

+ 214 - 0
desktop/core/ext-py/thrift-0.9.1/src/transport/TSSLSocket.py

@@ -0,0 +1,214 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+import socket
+import ssl
+
+from thrift.transport import TSocket
+from thrift.transport.TTransport import TTransportException
+
+
+class TSSLSocket(TSocket.TSocket):
+  """
+  SSL implementation of client-side TSocket
+
+  This class creates outbound sockets wrapped using the
+  python standard ssl module for encrypted connections.
+
+  The protocol used is set using the class variable
+  SSL_VERSION, which must be one of ssl.PROTOCOL_* and
+  defaults to  ssl.PROTOCOL_TLSv1 for greatest security.
+  """
+  SSL_VERSION = ssl.PROTOCOL_TLSv1
+
+  def __init__(self,
+               host='localhost',
+               port=9090,
+               validate=True,
+               ca_certs=None,
+               keyfile=None,
+               certfile=None,
+               unix_socket=None):
+    """Create SSL TSocket
+
+    @param validate: Set to False to disable SSL certificate validation
+    @type validate: bool
+    @param ca_certs: Filename to the Certificate Authority pem file, possibly a
+    file downloaded from: http://curl.haxx.se/ca/cacert.pem  This is passed to
+    the ssl_wrap function as the 'ca_certs' parameter.
+    @type ca_certs: str
+    @param keyfile: The private key
+    @type keyfile: str
+    @param certfile: The cert file
+    @type certfile: str
+    
+    Raises an IOError exception if validate is True and the ca_certs file is
+    None, not present or unreadable.
+    """
+    self.validate = validate
+    self.is_valid = False
+    self.peercert = None
+    if not validate:
+      self.cert_reqs = ssl.CERT_NONE
+    else:
+      self.cert_reqs = ssl.CERT_REQUIRED
+    self.ca_certs = ca_certs
+    self.keyfile = keyfile
+    self.certfile = certfile
+    if validate:
+      if ca_certs is None or not os.access(ca_certs, os.R_OK):
+        raise IOError('Certificate Authority ca_certs file "%s" '
+                      'is not readable, cannot validate SSL '
+                      'certificates.' % (ca_certs))
+    TSocket.TSocket.__init__(self, host, port, unix_socket)
+
+  def open(self):
+    try:
+      res0 = self._resolveAddr()
+      for res in res0:
+        sock_family, sock_type = res[0:2]
+        ip_port = res[4]
+        plain_sock = socket.socket(sock_family, sock_type)
+        self.handle = ssl.wrap_socket(plain_sock,
+                                      ssl_version=self.SSL_VERSION,
+                                      do_handshake_on_connect=True,
+                                      ca_certs=self.ca_certs,
+                                      keyfile=self.keyfile,
+                                      certfile=self.certfile,
+                                      cert_reqs=self.cert_reqs)
+        self.handle.settimeout(self._timeout)
+        try:
+          self.handle.connect(ip_port)
+        except socket.error, e:
+          if res is not res0[-1]:
+            continue
+          else:
+            raise e
+        break
+    except socket.error, e:
+      if self._unix_socket:
+        message = 'Could not connect to secure socket %s: %s' \
+                % (self._unix_socket, e)
+      else:
+        message = 'Could not connect to %s:%d: %s' % (self.host, self.port, e)
+      raise TTransportException(type=TTransportException.NOT_OPEN,
+                                message=message)
+    if self.validate:
+      self._validate_cert()
+
+  def _validate_cert(self):
+    """internal method to validate the peer's SSL certificate, and to check the
+    commonName of the certificate to ensure it matches the hostname we
+    used to make this connection.  Does not support subjectAltName records
+    in certificates.
+
+    raises TTransportException if the certificate fails validation.
+    """
+    cert = self.handle.getpeercert()
+    self.peercert = cert
+    if 'subject' not in cert:
+      raise TTransportException(
+        type=TTransportException.NOT_OPEN,
+        message='No SSL certificate found from %s:%s' % (self.host, self.port))
+    fields = cert['subject']
+    for field in fields:
+      # ensure structure we get back is what we expect
+      if not isinstance(field, tuple):
+        continue
+      cert_pair = field[0]
+      if len(cert_pair) < 2:
+        continue
+      cert_key, cert_value = cert_pair[0:2]
+      if cert_key != 'commonName':
+        continue
+      certhost = cert_value
+      # this check should be performed by some sort of Access Manager
+      if certhost == self.host:
+        # success, cert commonName matches desired hostname
+        self.is_valid = True
+        return
+      else:
+        raise TTransportException(
+          type=TTransportException.UNKNOWN,
+          message='Hostname we connected to "%s" doesn\'t match certificate '
+                  'provided commonName "%s"' % (self.host, certhost))
+    raise TTransportException(
+      type=TTransportException.UNKNOWN,
+      message='Could not validate SSL certificate from '
+              'host "%s".  Cert=%s' % (self.host, cert))
+
+
+class TSSLServerSocket(TSocket.TServerSocket):
+  """SSL implementation of TServerSocket
+
+  This uses the ssl module's wrap_socket() method to provide SSL
+  negotiated encryption.
+  """
+  SSL_VERSION = ssl.PROTOCOL_TLSv1
+
+  def __init__(self,
+               host=None,
+               port=9090,
+               certfile='cert.pem',
+               unix_socket=None):
+    """Initialize a TSSLServerSocket
+
+    @param certfile: filename of the server certificate, defaults to cert.pem
+    @type certfile: str
+    @param host: The hostname or IP to bind the listen socket to,
+                 i.e. 'localhost' for only allowing local network connections.
+                 Pass None to bind to all interfaces.
+    @type host: str
+    @param port: The port to listen on for inbound connections.
+    @type port: int
+    """
+    self.setCertfile(certfile)
+    TSocket.TServerSocket.__init__(self, host, port)
+
+  def setCertfile(self, certfile):
+    """Set or change the server certificate file used to wrap new connections.
+
+    @param certfile: The filename of the server certificate,
+                     i.e. '/etc/certs/server.pem'
+    @type certfile: str
+
+    Raises an IOError exception if the certfile is not present or unreadable.
+    """
+    if not os.access(certfile, os.R_OK):
+      raise IOError('No such certfile found: %s' % (certfile))
+    self.certfile = certfile
+
+  def accept(self):
+    plain_client, addr = self.handle.accept()
+    try:
+      client = ssl.wrap_socket(plain_client, certfile=self.certfile,
+                      server_side=True, ssl_version=self.SSL_VERSION)
+    except ssl.SSLError, ssl_exc:
+      # failed handshake/ssl wrap, close socket to client
+      plain_client.close()
+      # raise ssl_exc
+      # We can't raise the exception, because it kills most TServer derived
+      # serve() methods.
+      # Instead, return None, and let the TServer instance deal with it in
+      # other exception handling.  (but TSimpleServer dies anyway)
+      return None
+    result = TSocket.TSocket()
+    result.setHandle(client)
+    return result

+ 44 - 17
desktop/core/ext-py/thrift/src/transport/TSocket.py → desktop/core/ext-py/thrift-0.9.1/src/transport/TSocket.py

@@ -17,23 +17,33 @@
 # under the License.
 #
 
-from TTransport import *
-import os
 import errno
+import os
 import socket
+import sys
+
+from TTransport import *
+
 
 class TSocketBase(TTransportBase):
   def _resolveAddr(self):
     if self._unix_socket is not None:
-      return [(socket.AF_UNIX, socket.SOCK_STREAM, None, None, self._unix_socket)]
+      return [(socket.AF_UNIX, socket.SOCK_STREAM, None, None,
+               self._unix_socket)]
     else:
-      return socket.getaddrinfo(self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE | socket.AI_ADDRCONFIG)
+      return socket.getaddrinfo(self.host,
+                                self.port,
+                                socket.AF_UNSPEC,
+                                socket.SOCK_STREAM,
+                                0,
+                                socket.AI_PASSIVE | socket.AI_ADDRCONFIG)
 
   def close(self):
     if self.handle:
       self.handle.close()
       self.handle = None
 
+
 class TSocket(TSocketBase):
   """Socket implementation of TTransport base."""
 
@@ -45,7 +55,6 @@ class TSocket(TSocketBase):
     @param unix_socket(str)  The filename of a unix socket to connect to.
                              (host and port will be ignored.)
     """
-
     self.host = host
     self.port = port
     self.handle = None
@@ -56,15 +65,15 @@ class TSocket(TSocketBase):
     self.handle = h
 
   def isOpen(self):
-    return self.handle != None
+    return self.handle is not None
 
   def setTimeout(self, ms):
     if ms is None:
       self._timeout = None
     else:
-      self._timeout = ms/1000.0
+      self._timeout = ms / 1000.0
 
-    if (self.handle != None):
+    if self.handle is not None:
       self.handle.settimeout(self._timeout)
 
   def open(self):
@@ -86,34 +95,52 @@ class TSocket(TSocketBase):
         message = 'Could not connect to socket %s' % self._unix_socket
       else:
         message = 'Could not connect to %s:%d' % (self.host, self.port)
-      raise TTransportException(TTransportException.NOT_OPEN, message)
+      raise TTransportException(type=TTransportException.NOT_OPEN,
+                                message=message)
 
   def read(self, sz):
-    buff = self.handle.recv(sz)
+    try:
+      buff = self.handle.recv(sz)
+    except socket.error, e:
+      if (e.args[0] == errno.ECONNRESET and
+          (sys.platform == 'darwin' or sys.platform.startswith('freebsd'))):
+        # freebsd and Mach don't follow POSIX semantic of recv
+        # and fail with ECONNRESET if peer performed shutdown.
+        # See corresponding comment and code in TSocket::read()
+        # in lib/cpp/src/transport/TSocket.cpp.
+        self.close()
+        # Trigger the check to raise the END_OF_FILE exception below.
+        buff = ''
+      else:
+        raise
     if len(buff) == 0:
-      raise TTransportException('TSocket read 0 bytes')
+      raise TTransportException(type=TTransportException.END_OF_FILE,
+                                message='TSocket read 0 bytes')
     return buff
 
   def write(self, buff):
     if not self.handle:
-      raise TTransportException(TTransportException.NOT_OPEN, 'Transport not open')
+      raise TTransportException(type=TTransportException.NOT_OPEN,
+                                message='Transport not open')
     sent = 0
     have = len(buff)
     while sent < have:
       plus = self.handle.send(buff)
       if plus == 0:
-        raise TTransportException('TSocket sent 0 bytes')
+        raise TTransportException(type=TTransportException.END_OF_FILE,
+                                  message='TSocket sent 0 bytes')
       sent += plus
       buff = buff[plus:]
 
   def flush(self):
     pass
 
+
 class TServerSocket(TSocketBase, TServerTransportBase):
   """Socket implementation of TServerTransport base."""
 
-  def __init__(self, port=9090, unix_socket=None):
-    self.host = None
+  def __init__(self, host=None, port=9090, unix_socket=None):
+    self.host = host
     self.port = port
     self._unix_socket = unix_socket
     self.handle = None
@@ -137,8 +164,8 @@ class TServerSocket(TSocketBase, TServerTransportBase):
 
     self.handle = socket.socket(res[0], res[1])
     self.handle.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-    if hasattr(self.handle, 'set_timeout'):
-      self.handle.set_timeout(None)
+    if hasattr(self.handle, 'settimeout'):
+      self.handle.settimeout(None)
     self.handle.bind(res[4])
     self.handle.listen(128)
 

+ 21 - 17
desktop/core/ext-py/thrift/src/transport/TTransport.py → desktop/core/ext-py/thrift-0.9.1/src/transport/TTransport.py

@@ -18,11 +18,11 @@
 #
 
 from cStringIO import StringIO
-from struct import pack,unpack
+from struct import pack, unpack
 from thrift.Thrift import TException
 
-class TTransportException(TException):
 
+class TTransportException(TException):
   """Custom Transport Exception class"""
 
   UNKNOWN = 0
@@ -35,8 +35,8 @@ class TTransportException(TException):
     TException.__init__(self, message)
     self.type = type
 
-class TTransportBase:
 
+class TTransportBase:
   """Base class for Thrift transport layer."""
 
   def isOpen(self):
@@ -55,7 +55,7 @@ class TTransportBase:
     buff = ''
     have = 0
     while (have < sz):
-      chunk = self.read(sz-have)
+      chunk = self.read(sz - have)
       have += len(chunk)
       buff += chunk
 
@@ -70,6 +70,7 @@ class TTransportBase:
   def flush(self):
     pass
 
+
 # This class should be thought of as an interface.
 class CReadableTransport:
   """base class for transports that are readable from C"""
@@ -98,8 +99,8 @@ class CReadableTransport:
     """
     pass
 
-class TServerTransportBase:
 
+class TServerTransportBase:
   """Base class for Thrift server transports."""
 
   def listen(self):
@@ -111,15 +112,15 @@ class TServerTransportBase:
   def close(self):
     pass
 
-class TTransportFactoryBase:
 
+class TTransportFactoryBase:
   """Base class for a Transport Factory"""
 
   def getTransport(self, trans):
     return trans
 
-class TBufferedTransportFactory:
 
+class TBufferedTransportFactory:
   """Factory transport that builds buffered transports"""
 
   def getTransport(self, trans):
@@ -127,16 +128,19 @@ class TBufferedTransportFactory:
     return buffered
 
 
-class TBufferedTransport(TTransportBase,CReadableTransport):
-
-  """Class that wraps another transport and buffers its I/O."""
+class TBufferedTransport(TTransportBase, CReadableTransport):
+  """Class that wraps another transport and buffers its I/O.
 
+  The implementation uses a (configurable) fixed-size read buffer
+  but buffers all writes until a flush is performed.
+  """
   DEFAULT_BUFFER = 4096
 
-  def __init__(self, trans):
+  def __init__(self, trans, rbuf_size=DEFAULT_BUFFER):
     self.__trans = trans
     self.__wbuf = StringIO()
     self.__rbuf = StringIO("")
+    self.__rbuf_size = rbuf_size
 
   def isOpen(self):
     return self.__trans.isOpen()
@@ -152,7 +156,7 @@ class TBufferedTransport(TTransportBase,CReadableTransport):
     if len(ret) != 0:
       return ret
 
-    self.__rbuf = StringIO(self.__trans.read(max(sz, self.DEFAULT_BUFFER)))
+    self.__rbuf = StringIO(self.__trans.read(max(sz, self.__rbuf_size)))
     return self.__rbuf.read(sz)
 
   def write(self, buf):
@@ -172,9 +176,9 @@ class TBufferedTransport(TTransportBase,CReadableTransport):
 
   def cstringio_refill(self, partialread, reqlen):
     retstring = partialread
-    if reqlen < self.DEFAULT_BUFFER:
+    if reqlen < self.__rbuf_size:
       # try to make a read of as much as we can.
-      retstring += self.__trans.read(self.DEFAULT_BUFFER)
+      retstring += self.__trans.read(self.__rbuf_size)
 
     # but make sure we do read reqlen bytes.
     if len(retstring) < reqlen:
@@ -183,6 +187,7 @@ class TBufferedTransport(TTransportBase,CReadableTransport):
     self.__rbuf = StringIO(retstring)
     return self.__rbuf
 
+
 class TMemoryBuffer(TTransportBase, CReadableTransport):
   """Wraps a cStringIO object as a TTransport.
 
@@ -232,8 +237,8 @@ class TMemoryBuffer(TTransportBase, CReadableTransport):
     # only one shot at reading...
     raise EOFError()
 
-class TFramedTransportFactory:
 
+class TFramedTransportFactory:
   """Factory transport that builds framed transports"""
 
   def getTransport(self, trans):
@@ -242,7 +247,6 @@ class TFramedTransportFactory:
 
 
 class TFramedTransport(TTransportBase, CReadableTransport):
-
   """Class that wraps another transport and frames its I/O when writing."""
 
   def __init__(self, trans,):
@@ -298,7 +302,7 @@ class TFramedTransport(TTransportBase, CReadableTransport):
     # ask for a refill until the previous buffer is empty.  Therefore,
     # we can start reading new frames immediately.
     while len(prefix) < reqlen:
-      readFrame()
+      self.readFrame()
       prefix += self.__rbuf.getvalue()
     self.__rbuf = StringIO(prefix)
     return self.__rbuf

+ 43 - 3
desktop/core/ext-py/thrift/src/transport/TTwisted.py → desktop/core/ext-py/thrift-0.9.1/src/transport/TTwisted.py

@@ -16,16 +16,18 @@
 # specific language governing permissions and limitations
 # under the License.
 #
+
+from cStringIO import StringIO
+
 from zope.interface import implements, Interface, Attribute
 from twisted.internet.protocol import Protocol, ServerFactory, ClientFactory, \
     connectionDone
 from twisted.internet import defer
 from twisted.protocols import basic
 from twisted.python import log
-
+from twisted.web import server, resource, http
 
 from thrift.transport import TTransport
-from cStringIO import StringIO
 
 
 class TMessageSenderTransport(TTransport.TTransportBase):
@@ -79,7 +81,7 @@ class ThriftClientProtocol(basic.Int32StringReceiver):
         self.started.callback(self.client)
 
     def connectionLost(self, reason=connectionDone):
-        for k,v in self.client._reqs.iteritems():
+        for k, v in self.client._reqs.iteritems():
             tex = TTransport.TTransportException(
                 type=TTransport.TTransportException.END_OF_FILE,
                 message='Connection closed')
@@ -179,3 +181,41 @@ class ThriftClientFactory(ClientFactory):
             self.oprot_factory)
         p.factory = self
         return p
+
+
+class ThriftResource(resource.Resource):
+
+    allowedMethods = ('POST',)
+
+    def __init__(self, processor, inputProtocolFactory,
+        outputProtocolFactory=None):
+        resource.Resource.__init__(self)
+        self.inputProtocolFactory = inputProtocolFactory
+        if outputProtocolFactory is None:
+            self.outputProtocolFactory = inputProtocolFactory
+        else:
+            self.outputProtocolFactory = outputProtocolFactory
+        self.processor = processor
+
+    def getChild(self, path, request):
+        return self
+
+    def _cbProcess(self, _, request, tmo):
+        msg = tmo.getvalue()
+        request.setResponseCode(http.OK)
+        request.setHeader("content-type", "application/x-thrift")
+        request.write(msg)
+        request.finish()
+
+    def render_POST(self, request):
+        request.content.seek(0, 0)
+        data = request.content.read()
+        tmi = TTransport.TMemoryBuffer(data)
+        tmo = TTransport.TMemoryBuffer()
+
+        iprot = self.inputProtocolFactory.getProtocol(tmi)
+        oprot = self.outputProtocolFactory.getProtocol(tmo)
+
+        d = self.processor.process(iprot, oprot)
+        d.addCallback(self._cbProcess, request, tmo)
+        return server.NOT_DONE_YET

+ 248 - 0
desktop/core/ext-py/thrift-0.9.1/src/transport/TZlibTransport.py

@@ -0,0 +1,248 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+"""TZlibTransport provides a compressed transport and transport factory
+class, using the python standard library zlib module to implement
+data compression.
+"""
+
+from __future__ import division
+import zlib
+from cStringIO import StringIO
+from TTransport import TTransportBase, CReadableTransport
+
+
+class TZlibTransportFactory(object):
+  """Factory transport that builds zlib compressed transports.
+
+  This factory caches the last single client/transport that it was passed
+  and returns the same TZlibTransport object that was created.
+
+  This caching means the TServer class will get the _same_ transport
+  object for both input and output transports from this factory.
+  (For non-threaded scenarios only, since the cache only holds one object)
+
+  The purpose of this caching is to allocate only one TZlibTransport where
+  only one is really needed (since it must have separate read/write buffers),
+  and makes the statistics from getCompSavings() and getCompRatio()
+  easier to understand.
+  """
+  # class scoped cache of last transport given and zlibtransport returned
+  _last_trans = None
+  _last_z = None
+
+  def getTransport(self, trans, compresslevel=9):
+    """Wrap a transport, trans, with the TZlibTransport
+    compressed transport class, returning a new
+    transport to the caller.
+
+    @param compresslevel: The zlib compression level, ranging
+    from 0 (no compression) to 9 (best compression).  Defaults to 9.
+    @type compresslevel: int
+
+    This method returns a TZlibTransport which wraps the
+    passed C{trans} TTransport derived instance.
+    """
+    if trans == self._last_trans:
+      return self._last_z
+    ztrans = TZlibTransport(trans, compresslevel)
+    self._last_trans = trans
+    self._last_z = ztrans
+    return ztrans
+
+
+class TZlibTransport(TTransportBase, CReadableTransport):
+  """Class that wraps a transport with zlib, compressing writes
+  and decompresses reads, using the python standard
+  library zlib module.
+  """
+  # Read buffer size for the python fastbinary C extension,
+  # the TBinaryProtocolAccelerated class.
+  DEFAULT_BUFFSIZE = 4096
+
+  def __init__(self, trans, compresslevel=9):
+    """Create a new TZlibTransport, wrapping C{trans}, another
+    TTransport derived object.
+
+    @param trans: A thrift transport object, i.e. a TSocket() object.
+    @type trans: TTransport
+    @param compresslevel: The zlib compression level, ranging
+    from 0 (no compression) to 9 (best compression).  Default is 9.
+    @type compresslevel: int
+    """
+    self.__trans = trans
+    self.compresslevel = compresslevel
+    self.__rbuf = StringIO()
+    self.__wbuf = StringIO()
+    self._init_zlib()
+    self._init_stats()
+
+  def _reinit_buffers(self):
+    """Internal method to initialize/reset the internal StringIO objects
+    for read and write buffers.
+    """
+    self.__rbuf = StringIO()
+    self.__wbuf = StringIO()
+
+  def _init_stats(self):
+    """Internal method to reset the internal statistics counters
+    for compression ratios and bandwidth savings.
+    """
+    self.bytes_in = 0
+    self.bytes_out = 0
+    self.bytes_in_comp = 0
+    self.bytes_out_comp = 0
+
+  def _init_zlib(self):
+    """Internal method for setting up the zlib compression and
+    decompression objects.
+    """
+    self._zcomp_read = zlib.decompressobj()
+    self._zcomp_write = zlib.compressobj(self.compresslevel)
+
+  def getCompRatio(self):
+    """Get the current measured compression ratios (in,out) from
+    this transport.
+
+    Returns a tuple of:
+    (inbound_compression_ratio, outbound_compression_ratio)
+
+    The compression ratios are computed as:
+        compressed / uncompressed
+
+    E.g., data that compresses by 10x will have a ratio of: 0.10
+    and data that compresses to half of ts original size will
+    have a ratio of 0.5
+
+    None is returned if no bytes have yet been processed in
+    a particular direction.
+    """
+    r_percent, w_percent = (None, None)
+    if self.bytes_in > 0:
+      r_percent = self.bytes_in_comp / self.bytes_in
+    if self.bytes_out > 0:
+      w_percent = self.bytes_out_comp / self.bytes_out
+    return (r_percent, w_percent)
+
+  def getCompSavings(self):
+    """Get the current count of saved bytes due to data
+    compression.
+
+    Returns a tuple of:
+    (inbound_saved_bytes, outbound_saved_bytes)
+
+    Note: if compression is actually expanding your
+    data (only likely with very tiny thrift objects), then
+    the values returned will be negative.
+    """
+    r_saved = self.bytes_in - self.bytes_in_comp
+    w_saved = self.bytes_out - self.bytes_out_comp
+    return (r_saved, w_saved)
+
+  def isOpen(self):
+    """Return the underlying transport's open status"""
+    return self.__trans.isOpen()
+
+  def open(self):
+    """Open the underlying transport"""
+    self._init_stats()
+    return self.__trans.open()
+
+  def listen(self):
+    """Invoke the underlying transport's listen() method"""
+    self.__trans.listen()
+
+  def accept(self):
+    """Accept connections on the underlying transport"""
+    return self.__trans.accept()
+
+  def close(self):
+    """Close the underlying transport,"""
+    self._reinit_buffers()
+    self._init_zlib()
+    return self.__trans.close()
+
+  def read(self, sz):
+    """Read up to sz bytes from the decompressed bytes buffer, and
+    read from the underlying transport if the decompression
+    buffer is empty.
+    """
+    ret = self.__rbuf.read(sz)
+    if len(ret) > 0:
+      return ret
+    # keep reading from transport until something comes back
+    while True:
+      if self.readComp(sz):
+        break
+    ret = self.__rbuf.read(sz)
+    return ret
+
+  def readComp(self, sz):
+    """Read compressed data from the underlying transport, then
+    decompress it and append it to the internal StringIO read buffer
+    """
+    zbuf = self.__trans.read(sz)
+    zbuf = self._zcomp_read.unconsumed_tail + zbuf
+    buf = self._zcomp_read.decompress(zbuf)
+    self.bytes_in += len(zbuf)
+    self.bytes_in_comp += len(buf)
+    old = self.__rbuf.read()
+    self.__rbuf = StringIO(old + buf)
+    if len(old) + len(buf) == 0:
+      return False
+    return True
+
+  def write(self, buf):
+    """Write some bytes, putting them into the internal write
+    buffer for eventual compression.
+    """
+    self.__wbuf.write(buf)
+
+  def flush(self):
+    """Flush any queued up data in the write buffer and ensure the
+    compression buffer is flushed out to the underlying transport
+    """
+    wout = self.__wbuf.getvalue()
+    if len(wout) > 0:
+      zbuf = self._zcomp_write.compress(wout)
+      self.bytes_out += len(wout)
+      self.bytes_out_comp += len(zbuf)
+    else:
+      zbuf = ''
+    ztail = self._zcomp_write.flush(zlib.Z_SYNC_FLUSH)
+    self.bytes_out_comp += len(ztail)
+    if (len(zbuf) + len(ztail)) > 0:
+      self.__wbuf = StringIO()
+      self.__trans.write(zbuf + ztail)
+    self.__trans.flush()
+
+  @property
+  def cstringio_buf(self):
+    """Implement the CReadableTransport interface"""
+    return self.__rbuf
+
+  def cstringio_refill(self, partialread, reqlen):
+    """Implement the CReadableTransport interface for refill"""
+    retstring = partialread
+    if reqlen < self.DEFAULT_BUFFSIZE:
+      retstring += self.read(self.DEFAULT_BUFFSIZE)
+    while len(retstring) < reqlen:
+      retstring += self.read(reqlen - len(retstring))
+    self.__rbuf = StringIO(retstring)
+    return self.__rbuf

+ 1 - 1
desktop/core/ext-py/thrift/src/transport/__init__.py → desktop/core/ext-py/thrift-0.9.1/src/transport/__init__.py

@@ -17,4 +17,4 @@
 # under the License.
 #
 
-__all__ = ['TTransport', 'TSocket', 'THttpClient']
+__all__ = ['TTransport', 'TSocket', 'THttpClient', 'TZlibTransport']

+ 0 - 205
desktop/core/ext-py/thrift/src/protocol/TProtocol.py

@@ -1,205 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-from thrift.Thrift import *
-
-class TProtocolException(TException):
-
-  """Custom Protocol Exception class"""
-
-  UNKNOWN = 0
-  INVALID_DATA = 1
-  NEGATIVE_SIZE = 2
-  SIZE_LIMIT = 3
-  BAD_VERSION = 4
-
-  def __init__(self, type=UNKNOWN, message=None):
-    TException.__init__(self, message)
-    self.type = type
-
-class TProtocolBase:
-
-  """Base class for Thrift protocol driver."""
-
-  def __init__(self, trans):
-    self.trans = trans
-
-  def writeMessageBegin(self, name, type, seqid):
-    pass
-
-  def writeMessageEnd(self):
-    pass
-
-  def writeStructBegin(self, name):
-    pass
-
-  def writeStructEnd(self):
-    pass
-
-  def writeFieldBegin(self, name, type, id):
-    pass
-
-  def writeFieldEnd(self):
-    pass
-
-  def writeFieldStop(self):
-    pass
-
-  def writeMapBegin(self, ktype, vtype, size):
-    pass
-
-  def writeMapEnd(self):
-    pass
-
-  def writeListBegin(self, etype, size):
-    pass
-
-  def writeListEnd(self):
-    pass
-
-  def writeSetBegin(self, etype, size):
-    pass
-
-  def writeSetEnd(self):
-    pass
-
-  def writeBool(self, bool):
-    pass
-
-  def writeByte(self, byte):
-    pass
-
-  def writeI16(self, i16):
-    pass
-
-  def writeI32(self, i32):
-    pass
-
-  def writeI64(self, i64):
-    pass
-
-  def writeDouble(self, dub):
-    pass
-
-  def writeString(self, str):
-    pass
-
-  def readMessageBegin(self):
-    pass
-
-  def readMessageEnd(self):
-    pass
-
-  def readStructBegin(self):
-    pass
-
-  def readStructEnd(self):
-    pass
-
-  def readFieldBegin(self):
-    pass
-
-  def readFieldEnd(self):
-    pass
-
-  def readMapBegin(self):
-    pass
-
-  def readMapEnd(self):
-    pass
-
-  def readListBegin(self):
-    pass
-
-  def readListEnd(self):
-    pass
-
-  def readSetBegin(self):
-    pass
-
-  def readSetEnd(self):
-    pass
-
-  def readBool(self):
-    pass
-
-  def readByte(self):
-    pass
-
-  def readI16(self):
-    pass
-
-  def readI32(self):
-    pass
-
-  def readI64(self):
-    pass
-
-  def readDouble(self):
-    pass
-
-  def readString(self):
-    pass
-
-  def skip(self, type):
-    if type == TType.STOP:
-      return
-    elif type == TType.BOOL:
-      self.readBool()
-    elif type == TType.BYTE:
-      self.readByte()
-    elif type == TType.I16:
-      self.readI16()
-    elif type == TType.I32:
-      self.readI32()
-    elif type == TType.I64:
-      self.readI64()
-    elif type == TType.DOUBLE:
-      self.readDouble()
-    elif type == TType.STRING:
-      self.readString()
-    elif type == TType.STRUCT:
-      name = self.readStructBegin()
-      while True:
-        (name, type, id) = self.readFieldBegin()
-        if type == TType.STOP:
-          break
-        self.skip(type)
-        self.readFieldEnd()
-      self.readStructEnd()
-    elif type == TType.MAP:
-      (ktype, vtype, size) = self.readMapBegin()
-      for i in range(size):
-        self.skip(ktype)
-        self.skip(vtype)
-      self.readMapEnd()
-    elif type == TType.SET:
-      (etype, size) = self.readSetBegin()
-      for i in range(size):
-        self.skip(etype)
-      self.readSetEnd()
-    elif type == TType.LIST:
-      (etype, size) = self.readListBegin()
-      for i in range(size):
-        self.skip(etype)
-      self.readListEnd()
-
-class TProtocolFactory:
-  def getProtocol(self, trans):
-    pass

+ 6 - 3
desktop/core/src/desktop/lib/gen-py/djangothrift_test_gen/TestService-remote

@@ -1,9 +1,11 @@
 #!/usr/bin/env python
 #
-# Autogenerated by Thrift
+# Autogenerated by Thrift Compiler (0.9.0)
 #
 # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
 #
+#  options string: py:new_style
+#
 
 import sys
 import pprint
@@ -18,7 +20,7 @@ from ttypes import *
 
 if len(sys.argv) <= 1 or sys.argv[1] == '--help':
   print ''
-  print 'Usage: ' + sys.argv[0] + ' [-h host:port] [-u url] [-f[ramed]] function [arg1 [arg2...]]'
+  print 'Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] function [arg1 [arg2...]]'
   print ''
   print 'Functions:'
   print '  i32 ping(i32 input)'
@@ -36,7 +38,8 @@ argi = 1
 if sys.argv[argi] == '-h':
   parts = sys.argv[argi+1].split(':')
   host = parts[0]
-  port = int(parts[1])
+  if len(parts) > 1:
+    port = int(parts[1])
   argi += 2
 
 if sys.argv[argi] == '-u':

+ 14 - 10
desktop/core/src/desktop/lib/gen-py/djangothrift_test_gen/TestService.py

@@ -1,10 +1,12 @@
 #
-# Autogenerated by Thrift
+# Autogenerated by Thrift Compiler (0.9.0)
 #
 # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
 #
+#  options string: py:new_style
+#
 
-from thrift.Thrift import *
+from thrift.Thrift import TType, TMessageType, TException, TApplicationException
 from ttypes import *
 from thrift.Thrift import TProcessor
 from thrift.transport import TTransport
@@ -27,7 +29,7 @@ class Iface(object):
 class Client(Iface):
   def __init__(self, iprot, oprot=None):
     self._iprot = self._oprot = iprot
-    if oprot != None:
+    if oprot is not None:
       self._oprot = oprot
     self._seqid = 0
 
@@ -57,7 +59,7 @@ class Client(Iface):
     result = ping_result()
     result.read(self._iprot)
     self._iprot.readMessageEnd()
-    if result.success != None:
+    if result.success is not None:
       return result.success
     raise TApplicationException(TApplicationException.MISSING_RESULT, "ping failed: unknown result");
 
@@ -135,14 +137,15 @@ class ping_args(object):
       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
       return
     oprot.writeStructBegin('ping_args')
-    if self.input != None:
+    if self.input is not None:
       oprot.writeFieldBegin('input', TType.I32, 1)
       oprot.writeI32(self.input)
       oprot.writeFieldEnd()
     oprot.writeFieldStop()
     oprot.writeStructEnd()
-    def validate(self):
-      return
+
+  def validate(self):
+    return
 
 
   def __repr__(self):
@@ -193,14 +196,15 @@ class ping_result(object):
       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
       return
     oprot.writeStructBegin('ping_result')
-    if self.success != None:
+    if self.success is not None:
       oprot.writeFieldBegin('success', TType.I32, 0)
       oprot.writeI32(self.success)
       oprot.writeFieldEnd()
     oprot.writeFieldStop()
     oprot.writeStructEnd()
-    def validate(self):
-      return
+
+  def validate(self):
+    return
 
 
   def __repr__(self):

+ 4 - 2
desktop/core/src/desktop/lib/gen-py/djangothrift_test_gen/constants.py

@@ -1,9 +1,11 @@
 #
-# Autogenerated by Thrift
+# Autogenerated by Thrift Compiler (0.9.0)
 #
 # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
 #
+#  options string: py:new_style
+#
 
-from thrift.Thrift import *
+from thrift.Thrift import TType, TMessageType, TException, TApplicationException
 from ttypes import *
 

+ 32 - 27
desktop/core/src/desktop/lib/gen-py/djangothrift_test_gen/ttypes.py

@@ -1,10 +1,12 @@
 #
-# Autogenerated by Thrift
+# Autogenerated by Thrift Compiler (0.9.0)
 #
 # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
 #
+#  options string: py:new_style
+#
 
-from thrift.Thrift import *
+from thrift.Thrift import TType, TMessageType, TException, TApplicationException
 
 from thrift.transport import TTransport
 from thrift.protocol import TBinaryProtocol, TProtocol
@@ -78,18 +80,19 @@ class TestStruct(object):
       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
       return
     oprot.writeStructBegin('TestStruct')
-    if self.a != None:
+    if self.a is not None:
       oprot.writeFieldBegin('a', TType.STRING, 1)
       oprot.writeString(self.a)
       oprot.writeFieldEnd()
-    if self.b != None:
+    if self.b is not None:
       oprot.writeFieldBegin('b', TType.I32, 2)
       oprot.writeI32(self.b)
       oprot.writeFieldEnd()
     oprot.writeFieldStop()
     oprot.writeStructEnd()
-    def validate(self):
-      return
+
+  def validate(self):
+    return
 
 
   def __repr__(self):
@@ -150,18 +153,19 @@ class TestNesting(object):
       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
       return
     oprot.writeStructBegin('TestNesting')
-    if self.nested_struct != None:
+    if self.nested_struct is not None:
       oprot.writeFieldBegin('nested_struct', TType.STRUCT, 1)
       self.nested_struct.write(oprot)
       oprot.writeFieldEnd()
-    if self.b != None:
+    if self.b is not None:
       oprot.writeFieldBegin('b', TType.I32, 2)
       oprot.writeI32(self.b)
       oprot.writeFieldEnd()
     oprot.writeFieldStop()
     oprot.writeStructEnd()
-    def validate(self):
-      return
+
+  def validate(self):
+    return
 
 
   def __repr__(self):
@@ -349,61 +353,61 @@ class TestManyTypes(object):
       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
       return
     oprot.writeStructBegin('TestManyTypes')
-    if self.a_bool != None:
+    if self.a_bool is not None:
       oprot.writeFieldBegin('a_bool', TType.BOOL, 1)
       oprot.writeBool(self.a_bool)
       oprot.writeFieldEnd()
-    if self.a_byte != None:
+    if self.a_byte is not None:
       oprot.writeFieldBegin('a_byte', TType.BYTE, 2)
       oprot.writeByte(self.a_byte)
       oprot.writeFieldEnd()
-    if self.a_i16 != None:
+    if self.a_i16 is not None:
       oprot.writeFieldBegin('a_i16', TType.I16, 3)
       oprot.writeI16(self.a_i16)
       oprot.writeFieldEnd()
-    if self.a_i32 != None:
+    if self.a_i32 is not None:
       oprot.writeFieldBegin('a_i32', TType.I32, 4)
       oprot.writeI32(self.a_i32)
       oprot.writeFieldEnd()
-    if self.a_i64 != None:
+    if self.a_i64 is not None:
       oprot.writeFieldBegin('a_i64', TType.I64, 5)
       oprot.writeI64(self.a_i64)
       oprot.writeFieldEnd()
-    if self.a_double != None:
+    if self.a_double is not None:
       oprot.writeFieldBegin('a_double', TType.DOUBLE, 6)
       oprot.writeDouble(self.a_double)
       oprot.writeFieldEnd()
-    if self.a_string != None:
+    if self.a_string is not None:
       oprot.writeFieldBegin('a_string', TType.STRING, 7)
       oprot.writeString(self.a_string)
       oprot.writeFieldEnd()
-    if self.a_binary != None:
+    if self.a_binary is not None:
       oprot.writeFieldBegin('a_binary', TType.STRING, 8)
       oprot.writeString(self.a_binary)
       oprot.writeFieldEnd()
-    if self.a_enum != None:
+    if self.a_enum is not None:
       oprot.writeFieldBegin('a_enum', TType.I32, 9)
       oprot.writeI32(self.a_enum)
       oprot.writeFieldEnd()
-    if self.a_struct != None:
+    if self.a_struct is not None:
       oprot.writeFieldBegin('a_struct', TType.STRUCT, 10)
       self.a_struct.write(oprot)
       oprot.writeFieldEnd()
-    if self.a_set != None:
+    if self.a_set is not None:
       oprot.writeFieldBegin('a_set', TType.SET, 11)
       oprot.writeSetBegin(TType.I32, len(self.a_set))
       for iter25 in self.a_set:
         oprot.writeI32(iter25)
       oprot.writeSetEnd()
       oprot.writeFieldEnd()
-    if self.a_list != None:
+    if self.a_list is not None:
       oprot.writeFieldBegin('a_list', TType.LIST, 12)
       oprot.writeListBegin(TType.STRUCT, len(self.a_list))
       for iter26 in self.a_list:
         iter26.write(oprot)
       oprot.writeListEnd()
       oprot.writeFieldEnd()
-    if self.a_map != None:
+    if self.a_map is not None:
       oprot.writeFieldBegin('a_map', TType.MAP, 13)
       oprot.writeMapBegin(TType.I32, TType.STRUCT, len(self.a_map))
       for kiter27,viter28 in self.a_map.items():
@@ -411,11 +415,11 @@ class TestManyTypes(object):
         viter28.write(oprot)
       oprot.writeMapEnd()
       oprot.writeFieldEnd()
-    if self.a_string_with_default != None:
+    if self.a_string_with_default is not None:
       oprot.writeFieldBegin('a_string_with_default', TType.STRING, 14)
       oprot.writeString(self.a_string_with_default)
       oprot.writeFieldEnd()
-    if self.a_string_list != None:
+    if self.a_string_list is not None:
       oprot.writeFieldBegin('a_string_list', TType.LIST, 15)
       oprot.writeListBegin(TType.STRING, len(self.a_string_list))
       for iter29 in self.a_string_list:
@@ -424,8 +428,9 @@ class TestManyTypes(object):
       oprot.writeFieldEnd()
     oprot.writeFieldStop()
     oprot.writeStructEnd()
-    def validate(self):
-      return
+
+  def validate(self):
+    return
 
 
   def __repr__(self):

+ 15 - 0
desktop/core/src/desktop/lib/python_util.py

@@ -18,6 +18,7 @@
 # Extra python utils
 
 import select
+import socket
 from django.utils.translation import ugettext as _
 from desktop import conf
 
@@ -80,3 +81,17 @@ def create_synchronous_io_multiplexer(timeout=0):
     except ImportError:
       pass
   return SelectSynchronousIOMultiplexer(timeout)
+
+
+def find_unused_port():
+  """
+  Finds a port that's available.
+  Unfortunately, this port may not be available by the time
+  the subprocess uses it, but this generally works.
+  """
+  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
+  sock.bind(('127.0.0.1', 0))
+  sock.listen(socket.SOMAXCONN)
+  _, port = sock.getsockname()
+  sock.close()
+  return port

+ 8 - 6
desktop/core/src/desktop/lib/thrift_util_test.py

@@ -30,6 +30,7 @@ from djangothrift_test_gen.ttypes import TestStruct, TestNesting, TestEnum, Test
 from djangothrift_test_gen import TestService
 
 import hadoop
+import python_util
 import thrift_util
 from thrift_util import jsonable2thrift, thrift2json
 
@@ -44,10 +45,11 @@ from nose.tools import assert_equal
 class SimpleThriftServer(object):
   """A simple thrift server impl"""
   def __init__(self):
-    self.port = hadoop.mini_cluster.find_unused_port()
+    self.port = python_util.find_unused_port()
     self.pid = 0
 
   def ping(self, in_val):
+    logging.info('ping')
     return in_val * 2
 
   def start_server_process(self):
@@ -68,7 +70,7 @@ class SimpleThriftServer(object):
     # Child process runs the thrift server loop
     try:
       processor = TestService.Processor(self)
-      transport = TSocket.TServerSocket(self.port)
+      transport = TSocket.TServerSocket('localhost', self.port)
       server = TServer.TThreadedServer(processor,
                                        transport,
                                        TBufferedTransportFactory(),
@@ -83,15 +85,15 @@ class SimpleThriftServer(object):
     while time.time() < deadline:
       logging.info("Waiting for service to come online")
       try:
-        ping_s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        ping_s.connect(('localhost', self.port))
+        ping_s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        ping_s.connect(('localhost', self.port, 0, 0))
         ping_s.close()
         return
       except:
         _, status = os.waitpid(self.pid, os.WNOHANG)
         if status != 0:
           logging.info("SimpleThriftServer child process exited with %s" % (status,))
-        time.sleep(0.1)
+        time.sleep(5)
 
     logging.info("SimpleThriftServer took too long to come online")
     self.stop_server_process()
@@ -116,7 +118,7 @@ class TestWithThriftServer(object):
     cls.server = SimpleThriftServer()
     cls.server.start_server_process()
     cls.client = thrift_util.get_client(TestService.Client,
-                                        '127.0.0.1',
+                                        'localhost',
                                         cls.server.port,
                                         'Hue Unit Test Client',
                                         timeout_seconds=1)

+ 5 - 15
desktop/libs/hadoop/src/hadoop/mini_cluster.py

@@ -52,6 +52,8 @@ import simplejson
 import lxml.etree
 import urllib2
 
+from desktop.lib import python_util
+
 from hadoop.fs.hadoopfs import HadoopFileSystem
 from hadoop.job_tracker import LiveJobTracker
 import hadoop.cluster
@@ -82,18 +84,6 @@ TEST_USER_GROUP_MAPPING = {
 
 LOGGER=logging.getLogger(__name__)
 
-def find_unused_port():
-  """
-  Finds a port that's available.
-  Unfortunately, this port may not be available by the time
-  the subprocess uses it, but this generally works.
-  """
-  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
-  sock.bind(('127.0.0.1', 0))
-  sock.listen(socket.SOMAXCONN)
-  _, port = sock.getsockname()
-  sock.close()
-  return port
 
 class MiniHadoopCluster(object):
   """
@@ -178,13 +168,13 @@ rpc.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
         "-D", "jobclient.progress.monitor.poll.interval=100",
         "-D", "fs.checkpoint.period=1",
         # For a reason I don't fully understand, this must be 0.0.0.0 and not 'localhost'
-        "-D", "dfs.secondary.http.address=0.0.0.0:%d" % find_unused_port(),
+        "-D", "dfs.secondary.http.address=0.0.0.0:%d" % python_util.find_unused_port(),
         # We bind the NN's thrift interface to a port we find here.
         # This is suboptimal, since there's a race.  Alas, if we don't
         # do this here, the datanodes fail to discover the namenode's thrift
         # address, and there's a race there
-        "-D", "dfs.thrift.address=localhost:%d" % find_unused_port(),
-        "-D", "jobtracker.thrift.address=localhost:%d" % find_unused_port(),
+        "-D", "dfs.thrift.address=localhost:%d" % python_util.find_unused_port(),
+        "-D", "jobtracker.thrift.address=localhost:%d" % python_util.find_unused_port(),
         # Jobs realize they have finished faster with this timeout.
         "-D", "jobclient.completion.poll.interval=50",
         "-D", "hadoop.security.authorization=true",

+ 3 - 1
desktop/libs/hadoop/src/hadoop/pseudo_hdfs4.py

@@ -28,8 +28,10 @@ import tempfile
 import textwrap
 import time
 
+from desktop.lib.python_util import find_unused_port
+
 import hadoop
-from hadoop.mini_cluster import find_unused_port, write_config
+from hadoop.mini_cluster import write_config
 from hadoop.job_tracker import LiveJobTracker
 from desktop.lib.paths import get_run_root