utils.py 2.1 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980
  1. import datetime
  2. import decimal
  3. import re
  4. import json
  5. def write_json(path, data):
  6. with open(path, 'w', encoding='utf-8') as f:
  7. json.dump(data, f, ensure_ascii=False, indent=2)
  8. def read_json(path):
  9. with open(path, 'r', encoding='utf-8') as f:
  10. data = json.load(f)
  11. return data
  12. def read_text(filename)->str:
  13. data = []
  14. with open(filename, 'r', encoding='utf-8') as file:
  15. for line in file.readlines():
  16. line = line.strip()
  17. data.append(line)
  18. return data
  19. def save_raw_text(filename, content):
  20. with open(filename, 'w', encoding='utf-8') as file:
  21. file.write(content)
  22. def read_map_file(path):
  23. data = {}
  24. with open(path, 'r', encoding='utf-8') as f:
  25. for line in f.readlines():
  26. line = line.strip().split('\t')
  27. data[line[0]] = line[1].split('、')
  28. data[line[0]].append(line[0])
  29. return data
  30. def save_json(target_file,js,indent=4):
  31. with open(target_file, 'w', encoding='utf-8') as f:
  32. json.dump(js, f, ensure_ascii=False, indent=indent)
  33. def is_email(string):
  34. pattern = r'^[\w\.-]+@[\w\.-]+\.\w+$'
  35. match = re.match(pattern, string)
  36. if match:
  37. return True
  38. else:
  39. return False
  40. def examples_to_str(examples: list) -> list[str]:
  41. """
  42. from examples to a list of str
  43. """
  44. values = examples
  45. for i in range(len(values)):
  46. if isinstance(values[i], datetime.date):
  47. values = [values[i]]
  48. break
  49. elif isinstance(values[i], datetime.datetime):
  50. values = [values[i]]
  51. break
  52. elif isinstance(values[i], decimal.Decimal):
  53. values[i] = str(float(values[i]))
  54. elif is_email(str(values[i])):
  55. values = []
  56. break
  57. elif 'http://' in str(values[i]) or 'https://' in str(values[i]):
  58. values = []
  59. break
  60. elif values[i] is not None and not isinstance(values[i], str):
  61. pass
  62. elif values[i] is not None and '.com' in values[i]:
  63. pass
  64. return [str(v) for v in values if v is not None and len(str(v)) > 0]