tstpgnread.cc
1 // This program tests what is the best way to read a file
2 // with as goal to get to the end fast (maximum disk speed).
3 // That means that we want to read an integer number of
4 // blocks (likely 4096 bytes) at a time.
5 
6 #include "sys.h"
7 #include <iostream>
8 #include <fstream>
9 #include <cstdio>
10 #include <cstdlib>
11 #include <stdint.h>
12 #include <sys/time.h>
13 #include <sys/types.h>
14 #include <sys/stat.h>
15 #include <fcntl.h>
16 #include <unistd.h>
17 #include <gio/gio.h>
18 #include "debug.h"
19 
20 char const* filename="/opt/verylarge/chessgames/ftp.cis.uab.edu/crafty/enormous.pgn";
21 char const* warmupfile="/home/carlo/chess/fics2008.pgn";
22 
23 //#define REFERENCE_IMPLEMENTATION
24 //#define BUFFERSIZE_IMPLEMENTATION
25 //#define FSTREAM_READ_IMPLEMENTATION
26 //#define FSTREAM_GETLINE_IMPLEMENTATION
27 //#define GFILE_IMPLEMENTATION
28 //#define GFILE_ASYNC_IMPLEMENTATION
29 #define CWCHESS_PGN_IMPLEMENTATION
30 
31 #ifdef GFILE_ASYNC_IMPLEMENTATION
32 #include <gio/gio.h>
33 #include "PgnDatabase.h" // Needed for cwchess::pgn::DatabaseSeekable::S_buffer_size.
34 #endif
35 #ifdef CWCHESS_PGN_IMPLEMENTATION
36 #include <giomm/init.h>
37 #include "PgnDatabase.h"
38 #endif
39 #if defined(GFILE_ASYNC_IMPLEMENTATION) || defined(CWCHESS_PGN_IMPLEMENTATION)
40 std::ostream* global_os;
41 #endif
42 
43 struct timeval before, after;
44 
45 inline void start_timer(void)
46 {
47  gettimeofday(&before, NULL);
48 }
49 
50 uint64_t stop_timer(void)
51 {
52  gettimeofday(&after, NULL);
53  timersub(&after,& before,& after);
54  uint64_t t = after.tv_sec;
55  t* = 1000000;
56  t += after.tv_usec;
57  return t;
58 }
59 
60 char buf[4096 * 50];
61 
62 #ifdef REFERENCE_IMPLEMENTATION
63 void benchmark_reference(std::ostream& os, char const* filename)
64 {
65  int fd = open(filename, O_RDONLY);
66  size_t len = 0;
67  start_timer();
68  while(len < 945950820)
69  {
70  int s = read(fd, buf, sizeof(buf));
71  if (s == -1)
72  {
73  perror("read");
74  exit(0);
75  }
76  else if (s == 0)
77  break;
78  len += s;
79  }
80  uint64_t microseconds = stop_timer();
81  os << "Reference implementation (buffersize " << sizeof(buf) << "): " << microseconds << " microseconds." << std::endl;
82  close(fd);
83 }
84 #endif
85 
86 #ifdef BUFFERSIZE_IMPLEMENTATION
87 void benchmark_buffersize(std::ostream& os, char const* filename)
88 {
89  size_t const bufsize = 4096;
90  int fd = open(filename, O_RDONLY);
91  size_t len = 0;
92  start_timer();
93  while(len < 945950820)
94  {
95  int s = read(fd, buf, bufsize);
96  if (s == -1)
97  {
98  perror("read");
99  exit(0);
100  }
101  else if (s == 0)
102  break;
103  len += s;
104  }
105  uint64_t microseconds = stop_timer();
106  os << "read(2) (buffersize " << bufsize << "): " << microseconds << " microseconds." << std::endl;
107  close(fd);
108 }
109 #endif
110 
111 #ifdef FSTREAM_READ_IMPLEMENTATION
112 void benchmark_fstream_read(std::ostream& os, char const* filename)
113 {
114  std::ifstream inputfile;
115  inputfile.open(filename);
116  start_timer();
117  size_t len = 0;
118  while(inputfile)
119  {
120  inputfile.read(buf, 4096);
121  len += inputfile.gcount();
122  }
123  uint64_t microseconds = stop_timer();
124  os << "std::ifstream::read (buffersize 4096): " << microseconds << " microseconds. Size read: " << len << std::endl;
125  inputfile.close();
126 }
127 #endif
128 
129 #ifdef FSTREAM_GETLINE_IMPLEMENTATION
130 void benchmark_fstream_getline(std::ostream& os, char const* filename)
131 {
132  std::ifstream inputfile;
133  inputfile.open(filename);
134  start_timer();
135  size_t len = 0;
136  std::string line;
137  while(getline(inputfile, line))
138  {
139  len += line.length() + 1;
140  }
141  uint64_t microseconds = stop_timer();
142  os << "std::getline(std::ifstream, ...): " << microseconds << " microseconds. Size read: " << len << std::endl;
143  inputfile.close();
144 }
145 #endif
146 
147 #ifdef GFILE_IMPLEMENTATION
148 void benchmark_gfile(std::ostream& os, char const* filename)
149 {
150  g_type_init();
151  GFile* file = g_file_new_for_path(filename);
152  GCancellable* cancellable = g_cancellable_new();
153  GError* error = NULL;
154  GFileInputStream* stream = g_file_read(file, cancellable,& error);
155  if (stream == NULL)
156  {
157  std::cerr << "g_file_read failed: " << error->message << std::endl;
158  exit(1);
159  }
160  start_timer();
161  size_t len = 0;
162  gssize res;
163  do
164  {
165  res = g_input_stream_read(G_INPUT_STREAM(stream), buf, 4096, cancellable,& error);
166  if (res == -1)
167  {
168  std::cerr << "g_input_stream_read: " << error->message << std::endl;
169  exit(1);
170  }
171  len += res;
172  }
173  while(res > 0);
174  uint64_t microseconds = stop_timer();
175  os << "g_input_stream_read (buffersize 4096): " << microseconds << " microseconds. Size read: " << len << std::endl;
176  g_object_unref(stream);
177  g_object_unref(cancellable);
178  g_object_unref(file);
179 }
180 #endif
181 
182 #ifdef GFILE_ASYNC_IMPLEMENTATION
183 GMainLoop* gmain_loop;
184 GCancellable* cancellable;
185 void async_ready_callback(GObject* source_object, GAsyncResult* async_res, gpointer user_data);
186 void async_ready_callback2(GObject* source_object, GAsyncResult* async_res, gpointer user_data);
187 
188 void benchmark_gfile_async(std::ostream& os, char const* filename)
189 {
190  g_type_init();
191  global_os =& os;
192  GFile* file = g_file_new_for_path(filename);
193  cancellable = g_cancellable_new();
194  g_file_read_async(file, G_PRIORITY_DEFAULT, cancellable, async_ready_callback, file);
195  gmain_loop = g_main_loop_new(NULL, false);
196  g_main_loop_run(gmain_loop);
197  g_object_unref(cancellable);
198  g_object_unref(file);
199 }
200 #endif
201 
202 #ifdef CWCHESS_PGN_IMPLEMENTATION
203 Glib::RefPtr<Glib::MainLoop> main_loop;
204 Glib::RefPtr<cwchess::pgn::Database> pgn_data_base;
205 void open_finished(size_t len);
206 
207 void benchmark_cwchess_pgn(std::ostream& os, char const* filename)
208 {
209  using namespace cwchess;
210  Gio::init();
211  global_os =& os;
212  pgn_data_base = pgn::DatabaseSeekable::open(filename, sigc::ptr_fun(&open_finished));
213  main_loop = Glib::MainLoop::create(false);
214  start_timer();
215  main_loop->run();
216 }
217 #endif
218 
219 void clear_disk_cache(void)
220 {
221  // Free pagecache.
222  system("sudo sh -c \"sync; echo 1 > /proc/sys/vm/drop_caches\"");
223  // Warm up (read libraries etc).
224  start_timer();
225  uint64_t microseconds = stop_timer();
226  std::ofstream dump("/dev/null");
227  //std::ostream& dump(std::cout);
228 #ifdef REFERENCE_IMPLEMENTATION
229  benchmark_reference(dump, warmupfile);
230 #endif
231 #ifdef BUFFERSIZE_IMPLEMENTATION
232  benchmark_buffersize(dump, warmupfile);
233 #endif
234 #ifdef FSTREAM_READ_IMPLEMENTATION
235  benchmark_fstream_read(dump, warmupfile);
236 #endif
237 #ifdef FSTREAM_GETLINE_IMPLEMENTATION
238  benchmark_fstream_getline(dump, warmupfile);
239 #endif
240 #ifdef GFILE_IMPLEMENTATION
241  benchmark_gfile(dump, warmupfile);
242 #endif
243 #ifdef GFILE_ASYNC_IMPLEMENTATION
244  benchmark_gfile_async(dump, warmupfile);
245 #endif
246 #ifdef CWCHESS_PGN_IMPLEMENTATION
247  benchmark_cwchess_pgn(dump, warmupfile);
248 #endif
249  dump.close();
250  // Sleep to let other running application catch up too.
251  sleep(1);
252 }
253 
254 int main()
255 {
256  if (!Glib::thread_supported())
257  Glib::thread_init();
258  Debug(NAMESPACE_DEBUG::init());
259 
260 #ifdef REFERENCE_IMPLEMENTATION
261  clear_disk_cache();
262  benchmark_reference(std::cout, filename);
263 #endif
264 
265 #ifdef BUFFERSIZE_IMPLEMENTATION
266  clear_disk_cache();
267  benchmark_buffersize(std::cout, filename);
268 #endif
269 
270 #ifdef FSTREAM_READ_IMPLEMENTATION
271  clear_disk_cache();
272  benchmark_fstream_read(std::cout, filename);
273 #endif
274 
275 #ifdef FSTREAM_GETLINE_IMPLEMENTATION
276  clear_disk_cache();
277  benchmark_fstream_getline(std::cout, filename);
278 #endif
279 
280 #ifdef GFILE_IMPLEMENTATION
281  clear_disk_cache();
282  benchmark_gfile(std::cout, filename);
283 #endif
284 
285 #ifdef GFILE_ASYNC_IMPLEMENTATION
286  clear_disk_cache();
287  benchmark_gfile_async(std::cout, filename);
288 #endif
289 
290 #ifdef CWCHESS_PGN_IMPLEMENTATION
291  clear_disk_cache();
292  benchmark_cwchess_pgn(std::cout, filename);
293 #endif
294 }
295 
296 #ifdef GFILE_ASYNC_IMPLEMENTATION
297 size_t len = 0;
298 
299 void async_ready_callback(GObject* source_object, GAsyncResult* async_res, gpointer user_data)
300 {
301  GFile* file = G_FILE(user_data);
302  GError* error = NULL;
303  GFileInputStream* stream = g_file_read_finish(file, async_res,& error);
304  if (stream == NULL)
305  {
306  std::cerr << "g_file_read_finish failed: " << error->message << std::endl;
307  exit(1);
308  }
309  start_timer();
310  len = 0;
311  g_input_stream_read_async(G_INPUT_STREAM(stream), buf, cwchess::pgn::DatabaseSeekable::S_buffer_size,
312  G_PRIORITY_DEFAULT, cancellable, async_ready_callback2, stream);
313 }
314 
315 void async_ready_callback2(GObject* source_object, GAsyncResult* async_res, gpointer user_data)
316 {
317  GInputStream* stream = G_INPUT_STREAM(user_data);
318  GError* error = NULL;
319  gssize res = g_input_stream_read_finish(stream, async_res,& error);
320  if (res == -1)
321  {
322  std::cerr << "g_input_stream_read_finish: " << error->message << std::endl;
323  exit(1);
324  }
325  len += res;
326  if (res > 0)
327  g_input_stream_read_async(G_INPUT_STREAM(stream), buf, cwchess::pgn::DatabaseSeekable::S_buffer_size,
328  G_PRIORITY_DEFAULT, cancellable, async_ready_callback2, stream);
329  else
330  {
331  uint64_t microseconds = stop_timer();
332  * global_os << "g_input_stream_read_async (buffersize " << cwchess::pgn::DatabaseSeekable::S_buffer_size << "): " <<
333  microseconds << " microseconds. Size read: " << len << std::endl;
334  g_object_unref(stream);
335  g_main_loop_quit(gmain_loop);
336  }
337 }
338 #endif
339 
340 #ifdef CWCHESS_PGN_IMPLEMENTATION
341 void open_finished(size_t len)
342 {
343  uint64_t microseconds = stop_timer();
344  * global_os << "cwchess::pgn::DatabaseSeekable (buffersize " << cwchess::pgn::DatabaseSeekable::S_buffer_size << "): " <<
345  microseconds << " microseconds. Size read: " << len << "; number of lines: " << pgn_data_base->number_of_lines() << "; number of characters: " <<
346  pgn_data_base->number_of_characters() << std::endl;
347  main_loop->quit();
348 }
349 #endif
cwchess
A namespace for all chess related objects that are not related to the GUI.
Definition: Array.h:39
PgnDatabase.h
This file contains the declaration of class pgn::Database.
Copyright © 2006 - 2010 Carlo Wood.  All rights reserved.