Browse Source

Fixing memory usage when writting animations

The options -b and -B specify a buffer size for images waiting to we writtent
on disk
Yann Weber 5 years ago
parent
commit
ab7876e8d9
1 changed files with 25 additions and 6 deletions
  1. 25
    6
      gte/__main__.py

+ 25
- 6
gte/__main__.py View File

113
 parser_gen.add_argument('--threads', '-t', type=int, metavar='N',
113
 parser_gen.add_argument('--threads', '-t', type=int, metavar='N',
114
                         default=os.cpu_count()-1,
114
                         default=os.cpu_count()-1,
115
                         help="Animation writting threads")
115
                         help="Animation writting threads")
116
+parser_gen.add_argument('--buffer-min', '-b', type=int, metavar='N',
117
+                        default=10,
118
+                        help="Start populating animation buffer when this \
119
+amount of images are in the buffer")
120
+parser_gen.add_argument('--buffer-max', '-B', type=int, metavar='N',
121
+                        default=100,
122
+                        help="Start a buffer flush when this amount of images \
123
+are in the buffer")
116
 
124
 
117
 args = parser.parse_args()
125
 args = parser.parse_args()
118
 
126
 
245
             im.save(fname)
253
             im.save(fname)
246
             logger.debug('Image %s saved in %.3fs' % (fname, 
254
             logger.debug('Image %s saved in %.3fs' % (fname, 
247
                                                       time.time() - st1))
255
                                                       time.time() - st1))
256
+            del(im)
248
 
257
 
249
         write_pool = Pool(args.threads)
258
         write_pool = Pool(args.threads)
250
         async_res = []
259
         async_res = []
251
         img_count = 0
260
         img_count = 0
261
+        img_total = args.steps if args.anim_div <=1 else args.steps / (args.anim_div - 1)
252
     w = World(args.world_height, args.world_width, gray=args.gray)
262
     w = World(args.world_height, args.world_width, gray=args.gray)
253
     prog = rpnlib.RpnExpr.from_string(args.prog)
263
     prog = rpnlib.RpnExpr.from_string(args.prog)
254
     turmits = [LivingTurmit(world=w, prog=prog, diag=not args.no_diagonals)
264
     turmits = [LivingTurmit(world=w, prog=prog, diag=not args.no_diagonals)
258
     start = time.time()
268
     start = time.time()
259
     for step in range(args.steps):
269
     for step in range(args.steps):
260
         if step % 512 == 1:
270
         if step % 512 == 1:
261
-            msg = 'Step %d/%d %dus/step fractdim=%.3f'
271
+            msg = 'Step %d/%d avg: %dus/step fractdim=%.3f'
262
             msg %= (step, args.steps,
272
             msg %= (step, args.steps,
263
                     ((time.time() - start)*1000000)//step//len(turmits),
273
                     ((time.time() - start)*1000000)//step//len(turmits),
264
                     w.fractdim())
274
                     w.fractdim())
273
             fname = os.path.join(args.output_dir, fname)
283
             fname = os.path.join(args.output_dir, fname)
274
             im = Image.fromarray(np.uint8(w._val))
284
             im = Image.fromarray(np.uint8(w._val))
275
             async_res.append(write_pool.apply_async(save_pool, (im, fname)))
285
             async_res.append(write_pool.apply_async(save_pool, (im, fname)))
276
-            while async_res[0].ready():
277
-                async_res[0].get()
278
-                async_res.pop(0)
286
+            if len(async_res) > args.buffer_max:
287
+                logger.debug('Flushing animation on disk to empty RAM')
288
+                while len(async_res) > args.buffer_min:
289
+                    while async_res[0].ready():
290
+                        async_res[0].get()
291
+                        async_res.pop(0)
292
+                msg = '%d/%d images written at avg %.0f img/s ETL : %ds'
293
+                img_prog = img_count - args.buffer_min
294
+                img_pers = img_prog/((time.time() - start))
295
+                etl = (img_total - img_prog) // img_pers
296
+                msg %= (img_prog, img_total, img_pers, etl)
297
+                logger.info(msg)
279
     
298
     
280
     if args.output_dir is not None:
299
     if args.output_dir is not None:
281
-        while async_res[0].ready():
300
+        while len(async_res) and async_res[0].ready():
282
             async_res[0].get()
301
             async_res[0].get()
283
-            sync_res.pop(0)
302
+            async_res.pop(0)
284
         write_pool.close()
303
         write_pool.close()
285
         logger.info('Waiting for animation to be written on disk')
304
         logger.info('Waiting for animation to be written on disk')
286
         img_st1 = None
305
         img_st1 = None

Loading…
Cancel
Save