1234567891011121314151617 |
- {
- "zero_optimization": {
- "stage": 3,
- "allgather_partitions": true,
- "allgather_bucket_size": 5e8,
- "overlap_comm": false,
- "reduce_scatter": true,
- "reduce_bucket_size": 5e8,
- "contiguous_gradients": false,
- "stage3_max_live_parameters": 1e9,
- "stage3_max_reuse_distance": 1e9,
- "stage3_prefetch_bucket_size": 5e8,
- "stage3_param_persistence_threshold": 1e6,
- "stage3_gather_16bit_weights_on_model_save": false,
- "sub_group_size": 1e12
- }
- }
|