io_utils.py 1.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748
  1. # -*- coding: utf-8 -*-
  2. """
  3. @author:XuMing(xuming624@qq.com)
  4. @description:
  5. """
  6. import json
  7. import os
  8. import pickle
  9. def load_pkl(pkl_path):
  10. """
  11. 加载词典文件
  12. :param pkl_path:
  13. :return:
  14. """
  15. with open(pkl_path, 'rb') as f:
  16. result = pickle.load(f)
  17. return result
  18. def save_pkl(vocab, pkl_path, overwrite=True):
  19. """
  20. 存储文件
  21. :param pkl_path:
  22. :param overwrite:
  23. :return:
  24. """
  25. if os.path.exists(pkl_path) and not overwrite:
  26. return
  27. with open(pkl_path, 'wb') as f:
  28. # pickle.dump(vocab, f, protocol=pickle.HIGHEST_PROTOCOL)
  29. pickle.dump(vocab, f, protocol=0)
  30. def load_json(json_path, encoding='utf-8'):
  31. with open(json_path, mode='r', encoding=encoding) as json_file:
  32. data = json.load(json_file)
  33. return data
  34. def save_json(data, json_path, mode='w', encoding='utf-8'):
  35. dir = os.path.dirname(os.path.abspath(json_path))
  36. if not os.path.exists(dir):
  37. print(dir)
  38. os.makedirs(dir)
  39. with open(json_path, mode=mode, encoding=encoding) as f:
  40. f.write(json.dumps(data, ensure_ascii=False, indent=4))