docgen.py 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227
  1. #%%
  2. import json
  3. import itertools
  4. import os
  5. import os.path
  6. import sys
  7. import subprocess as sp
  8. import collections
  9. from pprint import pprint
  10. from sphinx.ext.napoleon.docstring import GoogleDocstring
  11. from sphinx.ext.napoleon import Config
  12. napoleon_config=Config(napoleon_use_param=True,napoleon_use_rtype=True)
  13. json_path=os.path.abspath(sys.argv[1])
  14. out_path=os.path.abspath(sys.argv[2])
  15. roots=sys.argv[3:]
  16. print(f"Generating documentation for {json_path}...")
  17. with open(json_path) as f:
  18. j=json.load(f)
  19. print(f"Load done!")
  20. # sys.exit(0)
  21. # with open('x.json','w') as f:
  22. # json.dump(j,f,indent=2)
  23. # 2. Get the list of modules and create the documentation tree
  24. modules={k:v["path"] for k,v in j.items() if v["kind"]=="module"}
  25. prefix=os.path.commonprefix(list(modules.values()))
  26. parsed_modules=collections.defaultdict(set)
  27. # os.system("rm -rf stdlib/*")
  28. root=""
  29. for mid,module in modules.items():
  30. while module not in roots:
  31. directory,name=os.path.split(module)
  32. directory=os.path.relpath(directory,root) # remove the prefix
  33. os.makedirs(f"{out_path}/{directory}",exist_ok=True)
  34. if name.endswith('.codon'):
  35. name=name[:-6] # drop suffix
  36. if name!='__init__':
  37. parsed_modules[directory].add((name,mid))
  38. print(root,mid,module, '->',name)
  39. module=os.path.split(module)[0]
  40. print(f"Module read done!")
  41. for directory,modules in parsed_modules.items():
  42. module=directory.replace('/','.')
  43. with open(f'{out_path}/{directory}/index.rst','w') as f:
  44. if module!='.':
  45. print(f".. codon:module:: {module}\n",file=f)
  46. print(f"{module}",file=f)
  47. else:
  48. print("Standard Library Reference",file=f)
  49. print(f"========\n",file=f)
  50. print(".. toctree::\n",file=f)
  51. for m in sorted(set(m for m,_ in modules)):
  52. if os.path.isdir(f'{root}/{directory}/{m}'):
  53. print(f" {m}/index",file=f)
  54. else:
  55. print(f" {m}",file=f)
  56. print(f" - Done with directory tree")
  57. def parse_docstr(s,level=1):
  58. """Parse docstr s and indent it with level spaces"""
  59. lines=GoogleDocstring(s,napoleon_config).lines()
  60. if isinstance(lines,str): # Napoleon failed
  61. s=s.split('\n')
  62. while s and s[0]=='':
  63. s=s[1:]
  64. while s and s[-1]=='':
  65. s=s[:-1]
  66. if not s:
  67. return ''
  68. i=0
  69. indent=len(list(itertools.takewhile(lambda i:i==' ',s[0])))
  70. lines=[l[indent:] for l in s]
  71. return '\n'.join((' '*level)+l for l in lines)
  72. def parse_type(a):
  73. """Parse type signature"""
  74. if not a:
  75. return ''
  76. s=''
  77. if isinstance(a,list):
  78. head,tail=a[0],a[1:]
  79. else:
  80. head,tail=a,[]
  81. if head not in j:
  82. return '?'
  83. s+=j[head]["name"] if head[0].isdigit() else head
  84. if tail:
  85. for ti,t in enumerate(tail):
  86. s+="[" if not ti else ", "
  87. s+=parse_type(t)
  88. s+="]"
  89. return s
  90. def parse_fn(v,skip_self=False,skip_braces=False):
  91. """Parse function signature after the name"""
  92. s=""
  93. if 'generics' in v and v['generics']:
  94. s+=f'[{", ".join(v["generics"])}]'
  95. if not skip_braces:
  96. s+="("
  97. cnt=0
  98. for ai,a in enumerate(v['args']):
  99. if ai==0 and a["name"]=="self" and skip_self:
  100. continue
  101. s+="" if not cnt else ", "
  102. cnt+=1
  103. s+=f'{a["name"]}'
  104. if "type" in a:
  105. print(a)
  106. s+=" : "+parse_type(a["type"])
  107. if "default" in a:
  108. s+=" = "+a["default"]+""
  109. if not skip_braces:
  110. s+=')'
  111. if "ret" in v:
  112. s+=" -> "+parse_type(v["ret"])
  113. # if "extern" in v:
  114. # s += f" (_{v['extern']} function_)"
  115. # s += "\n"
  116. return s
  117. # 3. Create documentation for each module
  118. for directory,(name,mid) in {(d,m) for d,mm in parsed_modules.items() for m in mm}:
  119. module=directory.replace('/','.')+f".{name}"
  120. file,mode=f'{out_path}/{directory}/{name}.rst','w'
  121. if os.path.isdir(f'{root}/{directory}/{name}'):
  122. continue
  123. if name=='__init__':
  124. file,mode=f'{out_path}/{directory}/index.rst','a'
  125. with open(file,mode) as f:
  126. print(f".. codon:module:: {module}\n",file=f)
  127. print(f":codon:mod:`{module}`",file=f)
  128. print("-"*(len(module)+13)+"\n",file=f)
  129. directory_prefix=directory+'/' if directory!='.' else ''
  130. print(f"Source code: `{directory_prefix}{name}.codon <https://github.com/exaloop/codon/blob/master/stdlib/{directory}/{name}.codon>`_\n",file=f)
  131. if 'doc' in j[mid]:
  132. print(parse_docstr(j[mid]['doc']),file=f)
  133. for i in j[mid]['children']:
  134. v=j[i]
  135. if v['kind']=='class' and v['type']=='extension':
  136. v['name']=j[v['parent']]['name']
  137. if v['name'].startswith('_'):
  138. continue
  139. if v['kind']=='class':
  140. if v['name'].endswith('Error'):
  141. v["type"]="exception"
  142. f.write(f'.. codon:{v["type"]}:: {v["name"]}')
  143. if 'generics' in v and v['generics']:
  144. f.write(f'[{",".join(v["generics"])}]')
  145. elif v['kind']=='function':
  146. f.write(f'.. codon:function:: {v["name"]}{parse_fn(v)}')
  147. elif v['kind']=='variable':
  148. f.write(f'.. codon:data:: {v["name"]}')
  149. # if v['kind'] == 'class' and v['type'] == 'extension':
  150. # f.write(f'**`{getLink(v["parent"])}`**')
  151. # else:
  152. # f.write(f'{m}.**`{v["name"]}`**')
  153. f.write("\n")
  154. # f.write("\n")
  155. # if v['kind'] == 'function' and 'attrs' in v and v['attrs']:
  156. # f.write("**Attributes:**" + ', '.join(f'`{x}`' for x in v['attrs']))
  157. # f.write("\n")
  158. if 'doc' in v:
  159. f.write("\n"+parse_docstr(v['doc'])+"\n")
  160. f.write("\n")
  161. if v['kind']=='class':
  162. # if 'args' in v and any(c['name'][0] != '_' for c in v['args']):
  163. # f.write('#### Arguments:\n')
  164. # for c in v['args']:
  165. # if c['name'][0] == '_':
  166. # continue
  167. # f.write(f'- **`{c["name"]} : `**')
  168. # f.write(parse_type(c["type"]) + "\n")
  169. # if 'doc' in c:
  170. # f.write(parse_docstr(c['doc'], 1) + "\n")
  171. # f.write("\n")
  172. mt=[c for c in v['members'] if j[c]['kind']=='function']
  173. props=[c for c in mt if 'property' in j[c].get('attrs',[])]
  174. if props:
  175. print(' **Properties:**\n',file=f)
  176. for c in props:
  177. v=j[c]
  178. f.write(f' .. codon:attribute:: {v["name"]}\n')
  179. if 'doc' in v:
  180. f.write("\n"+parse_docstr(v['doc'],4)+"\n\n")
  181. f.write("\n")
  182. magics=[c for c in mt if len(j[c]['name'])>4 and j[c]['name'].startswith('__') and j[c]['name'].endswith('__')]
  183. if magics:
  184. print(' **Magic methods:**\n',file=f)
  185. for c in magics:
  186. v=j[c]
  187. f.write(f' .. codon:method:: {v["name"]}{parse_fn(v,True)}\n')
  188. f.write(' :noindex:\n')
  189. if 'doc' in v:
  190. f.write("\n"+parse_docstr(v['doc'],4)+"\n\n")
  191. f.write("\n")
  192. methods=[c for c in mt if j[c]['name'][0]!='_' and c not in props]
  193. if methods:
  194. print(' **Methods:**\n',file=f)
  195. for c in methods:
  196. v=j[c]
  197. f.write(f' .. codon:method:: {v["name"]}{parse_fn(v,True)}\n')
  198. if 'doc' in v:
  199. f.write("\n"+parse_docstr(v['doc'],4)+"\n\n")
  200. f.write("\n")
  201. f.write("\n\n")
  202. f.write("\n\n")
  203. print(f" - Done with modules")