Browse Source

HUE-8208 [importer] Support SFDC live preview of objects and data

Romain Rigaux 7 years ago
parent
commit
4549eeb7a2

+ 38 - 36
desktop/libs/indexer/src/indexer/api3.py

@@ -23,6 +23,7 @@ import StringIO
 
 from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext as _
+from simple_salesforce.api import Salesforce
 
 from desktop.lib import django_mako
 from desktop.lib.django_util import JsonResponse
@@ -135,22 +136,12 @@ def guess_format(request):
     if file_format['streamSelection'] == 'kafka':
       format_ = {"type": "csv", "fieldSeparator": ",", "hasHeader": True, "quoteChar": "\"", "recordSeparator": "\\n", 'topics': get_topics()}
     elif file_format['streamSelection'] == 'sfdc':
-      stream = StringIO.StringIO()
-      stream.write("""AccountId,Amount,CampaignId,CloseDate,CreatedById,CreatedDate,CurrentGenerators__c,DeliveryInstallationStatus__c,Description,ExpectedRevenue,Fiscal,FiscalQuarter,FiscalYear,ForecastCategory,ForecastCategoryName,HasOpenActivity,HasOpportunityLineItem,HasOverdueTask,Id,IsClosed,IsDeleted,IsPrivate,IsWon,LastActivityDate,LastModifiedById,LastModifiedDate,LastReferencedDate,LastViewedDate,LeadSource,MainCompetitors__c,Name,NextStep,OrderNumber__c,OwnerId,Pricebook2Id,Probability,StageName,SystemModstamp,TotalOpportunityQuantity,TrackingNumber__c,Type
-0014600000Ctb1UAAR,30000.0,,1489968000000,005460000012t2xAAA,1490625405000,,In progress,,30000.0,2015 2,2,2015,Closed,Closed,false,false,false,00646000003Ymn1AAC,true,false,false,true,,005460000012t2xAAA,1490625405000,,,Partner,,GenePoint SLA,,546512,005460000012t2xAAA,,100.0,Closed Won,1490625405000,,,Existing Customer - Upgrade
-0014600000Ctb1NAAR,17000.0,,1486080000000,005460000012t2xAAA,1490625405000,,,,1700.0,2017 1,1,2017,Pipeline,Pipeline,false,false,false,00646000003YmmoAAC,false,false,false,false,,005460000012t2xAAA,1491321348000,,,Purchased List,Honda,Dickenson Mobile Generators,,,005460000012t2xAAA,,10.0,Qualification,1491389012000,,,New Customer
-0014600000Ctb1PAAR,915000.0,,1488326400000,005460000012t2xAAA,1490625405000,John Deere,Completed,,915000.0,2015 2,2,2015,Closed,Closed,false,false,false,00646000003Ymn7AAC,true,false,false,true,,005460000012t2xAAA,1490625405000,,,Partner,"John Deere, Mitsubishi, Hawkpower",United Oil Refinery Generators,,744343,005460000012t2xAAA,,100.0,Closed Won,1491389012000,,830150301360,New Customer
-0014600000Ctb1UAAR,85000.0,,1485302400000,005460000012t2xAAA,1490625405000,Honda,Completed,,85000.0,2015 1,1,2015,Closed,Closed,false,false,false,00646000003YmmrAAC,true,false,false,true,,005460000012t2xAAA,1490625405000,,,Partner,Honda,GenePoint Standby Generator,,908676,005460000012t2xAAA,,100.0,Closed Won,1490625405000,,830150301420,New Customer
-0014600000Ctb1MAAR,100000.0,,1485734400000,005460000012t2xAAA,1490625405000,,,,10000.0,2015 1,1,2015,Pipeline,Pipeline,false,false,false,00646000003YmmyAAC,false,false,false,false,,005460000012t2xAAA,1490625405000,,,Phone Inquiry,,Pyramid Emergency Generators,,,005460000012t2xAAA,,10.0,Prospecting,1490625405000,,,""")
-
-      indexer = MorphlineIndexer(request.user, request.fs)
-      format_ = indexer.guess_format({
-        "file": {
-          "stream": stream,
-          "name": file_format['path']
-        }
-      })
-      _convert_format(format_)
+      sf = Salesforce(
+          username=file_format['streamUsername'],
+          password=file_format['streamPassword'],
+          security_token=file_format['streamToken']
+      )
+      format_ = {"type": "csv", "fieldSeparator": ",", "hasHeader": True, "quoteChar": "\"", "recordSeparator": "\\n", 'objects': [sobject['name'] for sobject in sf.restful('sobjects/')['sobjects'] if sobject['queryable']]}
 
   format_['status'] = 0
   return JsonResponse(format_)
@@ -240,33 +231,44 @@ def guess_field_types(request):
         'kafkaFieldNames': file_format.get('kafkaFieldNames', ''),
         'data': '\n'.join([','.join(['...'] * len(file_format.get('kafkaFieldTypes', '').split(',')))] * 5)
       }
-    elif file_format['streamSelection'] == 'sfdc':
-      data = """AccountId,Amount,CampaignId,CloseDate,CreatedById,CreatedDate,CurrentGenerators__c,DeliveryInstallationStatus__c,Description,ExpectedRevenue,Fiscal,FiscalQuarter,FiscalYear,ForecastCategory,ForecastCategoryName,HasOpenActivity,HasOpportunityLineItem,HasOverdueTask,Id,IsClosed,IsDeleted,IsPrivate,IsWon,LastActivityDate,LastModifiedById,LastModifiedDate,LastReferencedDate,LastViewedDate,LeadSource,MainCompetitors__c,Name,NextStep,OrderNumber__c,OwnerId,Pricebook2Id,Probability,StageName,SystemModstamp,TotalOpportunityQuantity,TrackingNumber__c,Type
-0014600000Ctb1UAAR,30000.0,,1489968000000,005460000012t2xAAA,1490625405000,,In progress,,30000.0,2015 2,2,2015,Closed,Closed,false,false,false,00646000003Ymn1AAC,true,false,false,true,,005460000012t2xAAA,1490625405000,,,Partner,,GenePoint SLA,,546512,005460000012t2xAAA,,100.0,Closed Won,1490625405000,,,Existing Customer - Upgrade
-0014600000Ctb1NAAR,17000.0,,1486080000000,005460000012t2xAAA,1490625405000,,,,1700.0,2017 1,1,2017,Pipeline,Pipeline,false,false,false,00646000003YmmoAAC,false,false,false,false,,005460000012t2xAAA,1491321348000,,,Purchased List,Honda,Dickenson Mobile Generators,,,005460000012t2xAAA,,10.0,Qualification,1491389012000,,,New Customer
-0014600000Ctb1PAAR,915000.0,,1488326400000,005460000012t2xAAA,1490625405000,John Deere,Completed,,915000.0,2015 2,2,2015,Closed,Closed,false,false,false,00646000003Ymn7AAC,true,false,false,true,,005460000012t2xAAA,1490625405000,,,Partner,"John Deere, Mitsubishi, Hawkpower",United Oil Refinery Generators,,744343,005460000012t2xAAA,,100.0,Closed Won,1491389012000,,830150301360,New Customer
-0014600000Ctb1UAAR,85000.0,,1485302400000,005460000012t2xAAA,1490625405000,Honda,Completed,,85000.0,2015 1,1,2015,Closed,Closed,false,false,false,00646000003YmmrAAC,true,false,false,true,,005460000012t2xAAA,1490625405000,,,Partner,Honda,GenePoint Standby Generator,,908676,005460000012t2xAAA,,100.0,Closed Won,1490625405000,,830150301420,New Customer
-0014600000Ctb1MAAR,100000.0,,1485734400000,005460000012t2xAAA,1490625405000,,,,10000.0,2015 1,1,2015,Pipeline,Pipeline,false,false,false,00646000003YmmyAAC,false,false,false,false,,005460000012t2xAAA,1490625405000,,,Phone Inquiry,,Pyramid Emergency Generators,,,005460000012t2xAAA,,10.0,Prospecting,1490625405000,,,"""
-
-    stream = StringIO.StringIO()
-    stream.write(data)
+      stream = StringIO.StringIO()
+      stream.write(data)
 
-    _convert_format(file_format["format"], inverse=True)
+      _convert_format(file_format["format"], inverse=True)
 
-    indexer = MorphlineIndexer(request.user, request.fs)
-    format_ = indexer.guess_field_types({
-      "file": {
-          "stream": stream,
-          "name": file_format['path']
-        },
-      "format": file_format['format']
-    })
+      indexer = MorphlineIndexer(request.user, request.fs)
+      format_ = indexer.guess_field_types({
+        "file": {
+            "stream": stream,
+            "name": file_format['path']
+          },
+        "format": file_format['format']
+      })
 
-    if file_format['streamSelection'] == 'kafka':
       type_mapping = dict(zip(file_format['kafkaFieldNames'].split(','), file_format['kafkaFieldTypes'].split(',')))
       for col in format_['columns']:
         col['keyType'] = type_mapping[col['name']]
         col['type'] = type_mapping[col['name']]
+    elif file_format['streamSelection'] == 'sfdc':
+      sf = Salesforce(
+          username=file_format['streamUsername'],
+          password=file_format['streamPassword'],
+          security_token=file_format['streamToken']
+      )
+      table_metadata = [{
+          'name': column['name'],
+          'type': column['type']
+        } for column in sf.restful('sobjects/%(streamObject)s/describe/' % file_format)['fields']
+      ]
+      query = 'SELECT %s FROM %s LIMIT 4' % (', '.join([col['name'] for col in table_metadata]), file_format['streamObject'])
+      print query
+      format_ = {
+        "sample": [row.values()[1:] for row in sf.query_all(query)['records']],
+        "columns": [
+            Field(col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict()
+            for col in table_metadata
+        ]
+       }
 
   return JsonResponse(format_)
 

+ 28 - 11
desktop/libs/indexer/src/indexer/templates/importer.mako

@@ -394,7 +394,7 @@ ${ assist.assistPanel() }
             <!-- ko if: createWizard.source.streamSelection() == 'sfdc' -->
               <div class="control-group">
                 <label class="control-label"><div>${ _('Username') }</div>
-                  <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.streamUsername">
+                  <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.streamUsername" placeholder="user@company.com">
                 </label>
 
                 <label class="control-label"><div>${ _('Password') }</div>
@@ -402,7 +402,7 @@ ${ assist.assistPanel() }
                 </label>
 
                 <label class="control-label"><div>${ _('Token') }</div>
-                  <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.streamToken">
+                  <input type="password" class="input-xxlarge" data-bind="value: createWizard.source.streamToken">
                 </label>
 
                 <label class="control-label"><div>${ _('End point URL') }</div>
@@ -410,10 +410,14 @@ ${ assist.assistPanel() }
                 </label>
 
                 <br/>
-
+                <!-- ko if: createWizard.source.streamUsername() && createWizard.source.streamPassword() && createWizard.source.streamToken() -->
                 <label class="control-label"><div>${ _('Object') }</div>
-                  <input type="text" class="input-xxlarge" data-bind="value: createWizard.source.streamObject">
+                  <select class="input-xxlarge" data-bind="options: createWizard.source.streamObjects,
+                         value: createWizard.source.streamObject,
+                         optionsCaption: 'Choose...'"
+                         placeholder="${ _('The SFDC object to import, e.g. Account, Opportunity') }"></select>
                 </label>
+                <!-- /ko -->
               </div>
             <!-- /ko -->
           <!-- /ko -->
@@ -1622,11 +1626,17 @@ ${ assist.assistPanel() }
       self.streamUsername = ko.observable('');
       self.streamPassword = ko.observable('');
       self.streamToken = ko.observable('');
-      self.streamEndpointUrl = ko.observable('');
+      self.streamToken.subscribe(function(newVal) {
+        if (newVal) {
+          wizard.guessFormat(); // Todo
+        }
+      });
+      self.streamEndpointUrl = ko.observable('https://login.salesforce.com/services/Soap/u/42.0');
+      self.streamObjects = ko.observableArray();
       self.streamObject = ko.observable('');
       self.hasStreamSelected = ko.pureComputed(function() {
         return (self.streamSelection() == 'kafka' && self.kafkaSelectedTopics()) ||
-           (self.streamSelection() == 'sfdc')
+           (self.streamSelection() == 'sfdc' && self.streamObject())
       });
       self.hasStreamSelected.subscribe(function(newValue) {
         if (newValue) {
@@ -1688,7 +1698,7 @@ ${ assist.assistPanel() }
                 self.streamPassword().length > 0 &&
                 self.streamToken().length > 0 &&
                 self.streamEndpointUrl().length > 0 &&
-                self.streamObject().length > 0
+                self.streamObject()
               ;
           }
         } else if (self.inputFormat() == 'rdbms') {
@@ -1832,7 +1842,7 @@ ${ assist.assistPanel() }
               if (wizard.source.streamSelection() == 'kafka') {
                 name += '.' + wizard.source.kafkaSelectedTopics();
               } else {
-                name += '.' + wizard.source.streamSelection();
+                name += '.' + wizard.source.streamObject();
               }
             } else if (wizard.source.path()) {
               name += '.' + wizard.source.path().split('/').pop().split('.')[0];
@@ -2118,10 +2128,17 @@ ${ assist.assistPanel() }
           } else {
             var newFormat = ko.mapping.fromJS(new FileType(resp['type'], resp));
             self.source.format(newFormat);
-            if (self.source.inputFormat() == 'stream' && self.source.streamSelection() == 'kafka') {
-              self.source.kafkaTopics(resp['topics']);
+            if (self.source.inputFormat() == 'stream') {
+              if (self.source.streamSelection() == 'kafka') {
+                self.source.kafkaTopics(resp['topics']);
+              } else if (self.source.streamSelection() == 'sfdc') {
+                self.source.streamObjects(resp['objects']);
+              }
+            }
+    
+            if (self.source.inputFormat() != 'stream' || self.source.streamSelection() != 'sfdc') {
+              self.guessFieldTypes();
             }
-            self.guessFieldTypes();
           }
 
           self.isGuessingFormat(false);