ldif.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455
  1. """
  2. ldif - generate and parse LDIF data (see RFC 2849)
  3. See http://www.python-ldap.org/ for details.
  4. $Id: ldif.py,v 1.56 2010/07/19 08:23:22 stroeder Exp $
  5. Python compability note:
  6. Tested with Python 2.0+, but should work with Python 1.5.2+.
  7. """
  8. __version__ = '2.3.12'
  9. __all__ = [
  10. # constants
  11. 'ldif_pattern',
  12. # functions
  13. 'AttrTypeandValueLDIF','CreateLDIF','ParseLDIF',
  14. # classes
  15. 'LDIFWriter',
  16. 'LDIFParser',
  17. 'LDIFRecordList',
  18. 'LDIFCopy',
  19. ]
  20. import urlparse,urllib,base64,re,types
  21. try:
  22. from cStringIO import StringIO
  23. except ImportError:
  24. from StringIO import StringIO
  25. attrtype_pattern = r'[\w;.-]+(;[\w_-]+)*'
  26. attrvalue_pattern = r'(([^,]|\\,)+|".*?")'
  27. attrtypeandvalue_pattern = attrtype_pattern + r'[ ]*=[ ]*' + attrvalue_pattern
  28. rdn_pattern = attrtypeandvalue_pattern + r'([ ]*\+[ ]*' + attrtypeandvalue_pattern + r')*[ ]*'
  29. dn_pattern = rdn_pattern + r'([ ]*,[ ]*' + rdn_pattern + r')*[ ]*'
  30. dn_regex = re.compile('^%s$' % dn_pattern)
  31. ldif_pattern = '^((dn(:|::) %(dn_pattern)s)|(%(attrtype_pattern)s(:|::) .*)$)+' % vars()
  32. MOD_OP_INTEGER = {
  33. 'add':0,'delete':1,'replace':2
  34. }
  35. MOD_OP_STR = {
  36. 0:'add',1:'delete',2:'replace'
  37. }
  38. CHANGE_TYPES = ['add','delete','modify','modrdn']
  39. valid_changetype_dict = {}
  40. for c in CHANGE_TYPES:
  41. valid_changetype_dict[c]=None
  42. def is_dn(s):
  43. """
  44. returns 1 if s is a LDAP DN
  45. """
  46. if s=='':
  47. return 1
  48. rm = dn_regex.match(s)
  49. return rm!=None and rm.group(0)==s
  50. SAFE_STRING_PATTERN = '(^(\000|\n|\r| |:|<)|[\000\n\r\200-\377]+|[ ]+$)'
  51. safe_string_re = re.compile(SAFE_STRING_PATTERN)
  52. def list_dict(l):
  53. """
  54. return a dictionary with all items of l being the keys of the dictionary
  55. """
  56. return dict([(i,None) for i in l])
  57. class LDIFWriter:
  58. """
  59. Write LDIF entry or change records to file object
  60. Copy LDIF input to a file output object containing all data retrieved
  61. via URLs
  62. """
  63. def __init__(self,output_file,base64_attrs=None,cols=76,line_sep='\n'):
  64. """
  65. output_file
  66. file object for output
  67. base64_attrs
  68. list of attribute types to be base64-encoded in any case
  69. cols
  70. Specifies how many columns a line may have before it's
  71. folded into many lines.
  72. line_sep
  73. String used as line separator
  74. """
  75. self._output_file = output_file
  76. self._base64_attrs = list_dict([a.lower() for a in (base64_attrs or [])])
  77. self._cols = cols
  78. self._line_sep = line_sep
  79. self.records_written = 0
  80. def _unfoldLDIFLine(self,line):
  81. """
  82. Write string line as one or more folded lines
  83. """
  84. # Check maximum line length
  85. line_len = len(line)
  86. if line_len<=self._cols:
  87. self._output_file.write(line)
  88. self._output_file.write(self._line_sep)
  89. else:
  90. # Fold line
  91. pos = self._cols
  92. self._output_file.write(line[0:min(line_len,self._cols)])
  93. self._output_file.write(self._line_sep)
  94. while pos<line_len:
  95. self._output_file.write(' ')
  96. self._output_file.write(line[pos:min(line_len,pos+self._cols-1)])
  97. self._output_file.write(self._line_sep)
  98. pos = pos+self._cols-1
  99. return # _unfoldLDIFLine()
  100. def _needs_base64_encoding(self,attr_type,attr_value):
  101. """
  102. returns 1 if attr_value has to be base-64 encoded because
  103. of special chars or because attr_type is in self._base64_attrs
  104. """
  105. return self._base64_attrs.has_key(attr_type.lower()) or \
  106. not safe_string_re.search(attr_value) is None
  107. def _unparseAttrTypeandValue(self,attr_type,attr_value):
  108. """
  109. Write a single attribute type/value pair
  110. attr_type
  111. attribute type
  112. attr_value
  113. attribute value
  114. """
  115. if self._needs_base64_encoding(attr_type,attr_value):
  116. # Encode with base64
  117. self._unfoldLDIFLine(':: '.join([attr_type,base64.encodestring(attr_value).replace('\n','')]))
  118. else:
  119. self._unfoldLDIFLine(': '.join([attr_type,attr_value]))
  120. return # _unparseAttrTypeandValue()
  121. def _unparseEntryRecord(self,entry):
  122. """
  123. entry
  124. dictionary holding an entry
  125. """
  126. attr_types = entry.keys()[:]
  127. attr_types.sort()
  128. for attr_type in attr_types:
  129. for attr_value in entry[attr_type]:
  130. self._unparseAttrTypeandValue(attr_type,attr_value)
  131. def _unparseChangeRecord(self,modlist):
  132. """
  133. modlist
  134. list of additions (2-tuple) or modifications (3-tuple)
  135. """
  136. mod_len = len(modlist[0])
  137. if mod_len==2:
  138. changetype = 'add'
  139. elif mod_len==3:
  140. changetype = 'modify'
  141. else:
  142. raise ValueError,"modlist item of wrong length"
  143. self._unparseAttrTypeandValue('changetype',changetype)
  144. for mod in modlist:
  145. if mod_len==2:
  146. mod_type,mod_vals = mod
  147. elif mod_len==3:
  148. mod_op,mod_type,mod_vals = mod
  149. self._unparseAttrTypeandValue(MOD_OP_STR[mod_op],mod_type)
  150. else:
  151. raise ValueError,"Subsequent modlist item of wrong length"
  152. if mod_vals:
  153. for mod_val in mod_vals:
  154. self._unparseAttrTypeandValue(mod_type,mod_val)
  155. if mod_len==3:
  156. self._output_file.write('-'+self._line_sep)
  157. def unparse(self,dn,record):
  158. """
  159. dn
  160. string-representation of distinguished name
  161. record
  162. Either a dictionary holding the LDAP entry {attrtype:record}
  163. or a list with a modify list like for LDAPObject.modify().
  164. """
  165. if not record:
  166. # Simply ignore empty records
  167. return
  168. # Start with line containing the distinguished name
  169. self._unparseAttrTypeandValue('dn',dn)
  170. # Dispatch to record type specific writers
  171. if isinstance(record,types.DictType):
  172. self._unparseEntryRecord(record)
  173. elif isinstance(record,types.ListType):
  174. self._unparseChangeRecord(record)
  175. else:
  176. raise ValueError, "Argument record must be dictionary or list"
  177. # Write empty line separating the records
  178. self._output_file.write(self._line_sep)
  179. # Count records written
  180. self.records_written = self.records_written+1
  181. return # unparse()
  182. def CreateLDIF(dn,record,base64_attrs=None,cols=76):
  183. """
  184. Create LDIF single formatted record including trailing empty line.
  185. This is a compability function. Use is deprecated!
  186. dn
  187. string-representation of distinguished name
  188. record
  189. Either a dictionary holding the LDAP entry {attrtype:record}
  190. or a list with a modify list like for LDAPObject.modify().
  191. base64_attrs
  192. list of attribute types to be base64-encoded in any case
  193. cols
  194. Specifies how many columns a line may have before it's
  195. folded into many lines.
  196. """
  197. f = StringIO()
  198. ldif_writer = LDIFWriter(f,base64_attrs,cols,'\n')
  199. ldif_writer.unparse(dn,record)
  200. s = f.getvalue()
  201. f.close()
  202. return s
  203. class LDIFParser:
  204. """
  205. Base class for a LDIF parser. Applications should sub-class this
  206. class and override method handle() to implement something meaningful.
  207. Public class attributes:
  208. records_read
  209. Counter for records processed so far
  210. """
  211. def _stripLineSep(self,s):
  212. """
  213. Strip trailing line separators from s, but no other whitespaces
  214. """
  215. if s[-2:]=='\r\n':
  216. return s[:-2]
  217. elif s[-1:]=='\n':
  218. return s[:-1]
  219. else:
  220. return s
  221. def __init__(
  222. self,
  223. input_file,
  224. ignored_attr_types=None,
  225. max_entries=0,
  226. process_url_schemes=None,
  227. line_sep='\n'
  228. ):
  229. """
  230. Parameters:
  231. input_file
  232. File-object to read the LDIF input from
  233. ignored_attr_types
  234. Attributes with these attribute type names will be ignored.
  235. max_entries
  236. If non-zero specifies the maximum number of entries to be
  237. read from f.
  238. process_url_schemes
  239. List containing strings with URLs schemes to process with urllib.
  240. An empty list turns off all URL processing and the attribute
  241. is ignored completely.
  242. line_sep
  243. String used as line separator
  244. """
  245. self._input_file = input_file
  246. self._max_entries = max_entries
  247. self._process_url_schemes = list_dict([s.lower() for s in (process_url_schemes or [])])
  248. self._ignored_attr_types = list_dict([a.lower() for a in (ignored_attr_types or [])])
  249. self._line_sep = line_sep
  250. self.records_read = 0
  251. def handle(self,dn,entry):
  252. """
  253. Process a single content LDIF record. This method should be
  254. implemented by applications using LDIFParser.
  255. """
  256. def _unfoldLDIFLine(self):
  257. """
  258. Unfold several folded lines with trailing space into one line
  259. """
  260. unfolded_lines = [ self._stripLineSep(self._line) ]
  261. self._line = self._input_file.readline()
  262. while self._line and self._line[0]==' ':
  263. unfolded_lines.append(self._stripLineSep(self._line[1:]))
  264. self._line = self._input_file.readline()
  265. return ''.join(unfolded_lines)
  266. def _parseAttrTypeandValue(self):
  267. """
  268. Parse a single attribute type and value pair from one or
  269. more lines of LDIF data
  270. """
  271. # Reading new attribute line
  272. unfolded_line = self._unfoldLDIFLine()
  273. # Ignore comments which can also be folded
  274. while unfolded_line and unfolded_line[0]=='#':
  275. unfolded_line = self._unfoldLDIFLine()
  276. if not unfolded_line or unfolded_line=='\n' or unfolded_line=='\r\n':
  277. return None,None
  278. try:
  279. colon_pos = unfolded_line.index(':')
  280. except ValueError:
  281. # Treat malformed lines without colon as non-existent
  282. return None,None
  283. attr_type = unfolded_line[0:colon_pos]
  284. # if needed attribute value is BASE64 decoded
  285. value_spec = unfolded_line[colon_pos:colon_pos+2]
  286. if value_spec=='::':
  287. # attribute value needs base64-decoding
  288. attr_value = base64.decodestring(unfolded_line[colon_pos+2:])
  289. elif value_spec==':<':
  290. # fetch attribute value from URL
  291. url = unfolded_line[colon_pos+2:].strip()
  292. attr_value = None
  293. if self._process_url_schemes:
  294. u = urlparse.urlparse(url)
  295. if self._process_url_schemes.has_key(u[0]):
  296. attr_value = urllib.urlopen(url).read()
  297. elif value_spec==':\r\n' or value_spec=='\n':
  298. attr_value = ''
  299. else:
  300. attr_value = unfolded_line[colon_pos+2:].lstrip()
  301. return attr_type,attr_value
  302. def parse(self):
  303. """
  304. Continously read and parse LDIF records
  305. """
  306. self._line = self._input_file.readline()
  307. while self._line and \
  308. (not self._max_entries or self.records_read<self._max_entries):
  309. # Reset record
  310. version = None; dn = None; changetype = None; modop = None; entry = {}
  311. attr_type,attr_value = self._parseAttrTypeandValue()
  312. while attr_type!=None and attr_value!=None:
  313. if attr_type=='dn':
  314. # attr type and value pair was DN of LDIF record
  315. if dn!=None:
  316. raise ValueError, 'Two lines starting with dn: in one record.'
  317. if not is_dn(attr_value):
  318. raise ValueError, 'No valid string-representation of distinguished name %s.' % (repr(attr_value))
  319. dn = attr_value
  320. elif attr_type=='version' and dn is None:
  321. version = 1
  322. elif attr_type=='changetype':
  323. # attr type and value pair was DN of LDIF record
  324. if dn is None:
  325. raise ValueError, 'Read changetype: before getting valid dn: line.'
  326. if changetype!=None:
  327. raise ValueError, 'Two lines starting with changetype: in one record.'
  328. if not valid_changetype_dict.has_key(attr_value):
  329. raise ValueError, 'changetype value %s is invalid.' % (repr(attr_value))
  330. changetype = attr_value
  331. elif attr_value!=None and \
  332. not self._ignored_attr_types.has_key(attr_type.lower()):
  333. # Add the attribute to the entry if not ignored attribute
  334. if entry.has_key(attr_type):
  335. entry[attr_type].append(attr_value)
  336. else:
  337. entry[attr_type]=[attr_value]
  338. # Read the next line within an entry
  339. attr_type,attr_value = self._parseAttrTypeandValue()
  340. if entry:
  341. # append entry to result list
  342. self.handle(dn,entry)
  343. self.records_read = self.records_read+1
  344. return # parse()
  345. class LDIFRecordList(LDIFParser):
  346. """
  347. Collect all records of LDIF input into a single list.
  348. of 2-tuples (dn,entry). It can be a memory hog!
  349. """
  350. def __init__(
  351. self,
  352. input_file,
  353. ignored_attr_types=None,max_entries=0,process_url_schemes=None
  354. ):
  355. """
  356. See LDIFParser.__init__()
  357. Additional Parameters:
  358. all_records
  359. List instance for storing parsed records
  360. """
  361. LDIFParser.__init__(self,input_file,ignored_attr_types,max_entries,process_url_schemes)
  362. self.all_records = []
  363. def handle(self,dn,entry):
  364. """
  365. Append single record to dictionary of all records.
  366. """
  367. self.all_records.append((dn,entry))
  368. class LDIFCopy(LDIFParser):
  369. """
  370. Copy LDIF input to LDIF output containing all data retrieved
  371. via URLs
  372. """
  373. def __init__(
  374. self,
  375. input_file,output_file,
  376. ignored_attr_types=None,max_entries=0,process_url_schemes=None,
  377. base64_attrs=None,cols=76,line_sep='\n'
  378. ):
  379. """
  380. See LDIFParser.__init__() and LDIFWriter.__init__()
  381. """
  382. LDIFParser.__init__(self,input_file,ignored_attr_types,max_entries,process_url_schemes)
  383. self._output_ldif = LDIFWriter(output_file,base64_attrs,cols,line_sep)
  384. def handle(self,dn,entry):
  385. """
  386. Write single LDIF record to output file.
  387. """
  388. self._output_ldif.unparse(dn,entry)
  389. def ParseLDIF(f,ignore_attrs=None,maxentries=0):
  390. """
  391. Parse LDIF records read from file.
  392. This is a compability function. Use is deprecated!
  393. """
  394. ldif_parser = LDIFRecordList(
  395. f,ignored_attr_types=ignore_attrs,max_entries=maxentries,process_url_schemes=0
  396. )
  397. ldif_parser.parse()
  398. return ldif_parser.all_records