· 8 years ago · Nov 17, 2017, 01:16 AM
1/*
2 * RATIONALE:
3 *
4 * This was written to show a pathological case with Ryzen where this
5 * test escapes Ryzens typical dual page walker hardware and cache utag
6 * mechanism.
7 *
8 * Software Optimization Guide for AMD Family 17h Processors section 2.7.3
9 * states "Linear aliasing occurs when two different linear addresses are
10 * mapped to the same physical address. This can cause performance penalties
11 * for loads and stores to the aliased cachelines. A load to an address
12 * that is valid in the L1 DC but under a different linear alias will see
13 * an L1 DC miss, which requires an L2 cache request to be made. The
14 * latency will generally be no larger than that of an L2 cache hit.
15 * However, if multiple aliased loads or stores are in-flight simultaneously,
16 * they each may experience L1 DC misses as they update the utag with a
17 * particular linear address and remove another linear address from being
18 * able to access the cacheline."
19 *
20 * In particular, this hits the latter case, a full L1 DC miss which then
21 * becomes a full 8-way lookup. In particular, if the physical tag is
22 * wrong, then it cannot look in the set since the set is virtual.
23 *
24 * In general, CPUs cannot afford to translate linear addresses to physical
25 * ones, then look it up in L1, so they optimistically lookup by virtual address
26 * instead and make sure later (after thr address translation is done
27 * concurrently) that the physical tag in the cache matches the TLB lookup.
28 * When you create an alias (two or more linear addresses which map to the
29 * same physical page(s), you cause this mismatch to happen.
30 *
31 * When these mismatches occur the CPU has to do a full page walk, and this
32 * is where Ryzen's dual page walk hardware causes a pathological
33 * performance issue, in particular, the dual page walker hardware is
34 * required to behave atomically since two threads (on the same core)
35 * are allowed to translate addresses concurrently, however this is only
36 * on a per-core basis. When you split up the walks across two (or more)
37 * physical cores the walking hardware serializes. This is because only
38 * one dual page walking assist exists. On other x86 systems there exists
39 * page walking hardware for each physical core.
40 *
41 * One should question why Ryzen has only one assist for this when page
42 * walking is trivial to do on x86 (you chase at most 4 physical addresses.)
43 * and the reason for this is because each assist needs a cache of page
44 * table entries at L1 DC and Ryzen chose to avoid wasting L1 DC. This
45 * decision makes sense since cost of cache in terms of die area and wires
46 * is exceptionally high as is and having to duplicate that for all
47 * the physical cores would drive the cost and power consumption of Ryzen
48 * up. You can afford to make that design decision when you have fewer
49 * cores, but alas that's not the case with something like the 1950x.
50 *
51 * ALGORITHM:
52 *
53 * The way this measures performance across CPUs is via the use of
54 * explicit shared memory mappings to ensure that each CPU gets it's
55 * own virtual mapping. The read end only gets read permissions while the
56 * write end only gets write permissions. This ensures separate page table
57 * entries.
58 *
59 * This was written with standard UNIX in mind so it uses a socket to
60 * hand over the shared memory file descriptor through the use of sendmsg
61 * opposed to just sending the file descriptor over directly which would
62 * work okay in Linux but no where else.
63 *
64 * The approach here uses a block of shared memory mapped in a read process
65 * and a write process constructed by this process. This memory is touched
66 * completely from beginning to end with simple load/store in each process.
67 * The size of the block of memory can be configured with the TOUCH macro.
68 *
69 * The entire process is done multiple times as to reduce scheduling noise
70 * and thread construction cost, this is configured with the TRIALS macro.
71 * The time is averaged at the end to provide a stablized result. Note:
72 * all time is measured in wall clock time and not CPU time as we also
73 * want to consider the cost of blocking operations like swap.
74 *
75 * On my 1950x this test takes ~21 minutes, whereas my i7-4790K takes
76 * only ~11 minutes.
77 *
78 * Copyright (C) 2017
79 * Dale Weiler
80 *
81 * Permission is hereby granted, free of charge, to any person obtaining a copy of
82 * this software and associated documentation files (the "Software"), to deal in
83 * the Software without restriction, including without limitation the rights to
84 * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
85 * of the Software, and to permit persons to whom the Software is furnished to do
86 * so, subject to the following conditions:
87 *
88 * The above copyright notice and this permission notice shall be included in all
89 * copies or substantial portions of the Software.
90 *
91 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
92 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
93 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
94 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
95 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
96 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
97 * SOFTWARE.
98 */
99#define _GNU_SOURCE
100
101#include <stdio.h>
102#include <stdlib.h>
103#include <string.h>
104#include <stdarg.h>
105#include <float.h>
106#include <limits.h>
107#include <assert.h>
108#include <pthread.h>
109
110#include <sched.h>
111#include <unistd.h>
112#include <sys/time.h>
113#include <sys/resource.h>
114#include <sys/mman.h>
115#include <sys/fcntl.h>
116#include <sys/socket.h>
117
118/* How many times to run the benchmark (Default) */
119#define TRIALS 128
120
121/* The benchmark memory to operate on (Default) */
122#define MEMORY (size_t)(32ull * 1024ull * 1024ull) // 32MiB
123
124/* Simple binary semaphore with pthread mutex and condition variable */
125struct bsem {
126 pthread_mutex_t m;
127 pthread_cond_t c;
128 int v;
129};
130
131static int bsem_init(struct bsem* b) {
132 if (pthread_mutex_init(&b->m, NULL) < 0) {
133 return -1;
134 }
135
136 if (pthread_cond_init(&b->c, NULL) < 0) {
137 pthread_mutex_destroy(&b->m);
138 return -1;
139 }
140
141 b->v = 0;
142 return 0;
143}
144
145static void bsem_destroy(struct bsem* b) {
146 pthread_mutex_destroy(&b->m);
147 pthread_cond_destroy(&b->c);
148}
149
150int bsem_post(struct bsem* b) {
151 if (pthread_mutex_lock(&b->m) < 0) {
152 return -1;
153 }
154 b->v++;
155 pthread_cond_signal(&b->c);
156 pthread_mutex_unlock(&b->m);
157 return 0;
158}
159
160static int bsem_wait(struct bsem* b) {
161 if (pthread_mutex_lock(&b->m) < 0) {
162 return -1;
163 }
164 while (!b->v) {
165 pthread_cond_wait(&b->c, &b->m);
166 }
167 b->v--;
168 pthread_mutex_unlock(&b->m);
169 return 0;
170}
171
172struct threaddata
173{
174 /* The amount of threads per CPU */
175 int threads;
176 /* Semaphore to communicate that thread is started */
177 struct bsem* sem;
178 /* Result of the thread, should be equal to TOUCH when thread joins */
179 size_t result;
180 /* The amount of working memory to touch */
181 size_t memory;
182 /* The read socket to get the shared memory file descriptor */
183 int socket;
184};
185
186static void err(const char *fmt, ...) {
187 va_list va;
188 va_start(va, fmt);
189 vfprintf(stderr, fmt, va);
190 va_end(va);
191 fflush(stderr);
192}
193
194/* Can't safely send shared memory file descriptors as integers
195 * across processes, tho here it's safe on Linux since threads are not
196 * really processes. The translation level is needed to be safe and does
197 * not hurt on Linux.
198 */
199static int sendfd(int sock, int fd) {
200 char buf[CMSG_SPACE(sizeof fd)];
201 memset(buf, '\0', sizeof buf);
202
203 struct iovec io = {
204 .iov_base = "",
205 .iov_len = 1
206 };
207
208 struct msghdr msg = {
209 .msg_iov = &io,
210 .msg_iovlen = 1,
211 .msg_control = buf,
212 .msg_controllen = sizeof buf
213 };
214
215 struct cmsghdr* cmsg = CMSG_FIRSTHDR(&msg);
216 cmsg->cmsg_level = SOL_SOCKET;
217 cmsg->cmsg_type = SCM_RIGHTS;
218 cmsg->cmsg_len = CMSG_LEN(sizeof fd);
219
220 memmove(CMSG_DATA(cmsg), &fd, sizeof fd);
221
222 msg.msg_controllen = cmsg->cmsg_len;
223
224 return sendmsg(sock, &msg, 0) >= 0;
225}
226
227static int readfd(int sock) {
228 char mbuf[1]; /* Message buffer */
229 char cbuf[128]; /* Control buffer */
230
231 struct iovec io = {
232 .iov_base = mbuf,
233 .iov_len = sizeof mbuf
234 };
235
236 struct msghdr msg = {
237 .msg_iov = &io,
238 .msg_iovlen = sizeof mbuf,
239 .msg_control = cbuf,
240 .msg_controllen = sizeof cbuf
241 };
242
243 if (recvmsg(sock, &msg, 0) < 0) {
244 return -1;
245 }
246
247 struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
248 int fd;
249 memmove(&fd, CMSG_DATA(cmsg), sizeof fd);
250 return fd;
251}
252
253static int setcpuandprio(const char* which, int n) {
254 cpu_set_t set;
255 CPU_ZERO(&set);
256 CPU_SET(n, &set);
257 if (pthread_setaffinity_np(pthread_self(), sizeof set, &set) < 0) {
258 err("failed to set thread '%s' to cpu '%d'\n", which, n);
259 return -1;
260 }
261 struct sched_param p = {
262 .sched_priority = 99 /* Max priority */
263 };
264 if (pthread_setschedparam(pthread_self(), SCHED_FIFO, &p) < 0) {
265 err("failed to set thread '%s' priority", which);
266 return -1;
267 }
268 return 0;
269}
270
271/* Functions to prevent the compiler from removing or reordering loads and stores
272 * a function call behaves as a full memory barrier too
273 */
274static unsigned char __attribute__ ((noinline)) load(unsigned char *x) {
275 return *x;
276}
277
278static void __attribute__ ((noinline)) store(unsigned char *x, unsigned char y) {
279 *x = y;
280}
281
282/* Read thread */
283static void* rd(void* opaque)
284{
285 struct threaddata *data = opaque;
286
287 /* Set the CPU this thread will run on */
288 if (setcpuandprio("rd", 0) < 0) {
289 return NULL;
290 }
291
292 /* Get the shared memory file descriptor */
293 int fd = readfd(data->socket);
294 if (fcntl(fd, F_GETFD) < 0) {
295 err("rd got invalid file descriptor for shared memory\n");
296 goto error;
297 }
298
299 /* Map in shared memory */
300 unsigned char* touch = mmap(0, data->memory, PROT_READ, MAP_SHARED, fd, 0);
301 if (touch == MAP_FAILED) {
302 err("failed to map shared memory for rd thread\n");
303 goto error;
304 }
305
306 /* Inform the parent process we're ready */
307 bsem_post(data->sem);
308
309 /* Do a ton of reads */
310 for (size_t i = 0; i < data->memory; i++) {
311 load(touch + i);
312 data->result++;
313 }
314
315 /* No longer need shared memory mapping */
316 munmap(touch, data->memory);
317 close(fd);
318
319 return NULL;
320
321error:
322 if (fd) {
323 close(fd);
324 }
325 /* Don't dead lock in main during initialization */
326 bsem_post(data->sem);
327 return NULL;
328}
329
330/* Write thread */
331static void* wr(void* opaque)
332{
333 struct threaddata *data = opaque;
334
335 /* Set the CPU this thread will run on */
336 if (setcpuandprio("wr", data->threads) < 0) {
337 return NULL;
338 }
339
340 /* Get the shared memory file descriptor */
341 int fd = readfd(data->socket);
342 if (fcntl(fd, F_GETFD) < 0) {
343 err("wr got invalid file descriptor for shared memory\n");
344 goto error;
345 }
346
347 /* Map in shared memory */
348 unsigned char* touch = mmap(0, data->memory, PROT_WRITE, MAP_SHARED, fd, 0);
349 if (touch == MAP_FAILED) {
350 err("failed to map shared memory for wr thread\n");
351 goto error;
352 }
353
354 /* Inform the parent process we're ready */
355 bsem_post(data->sem);
356
357 /* Do a ton of writes */
358 for (size_t i = 0; i < data->memory; i++) {
359 store(touch + i, i);
360 data->result++;
361 }
362
363 /* No longer need shared memory mapping */
364 munmap(touch, data->memory);
365 close(fd);
366
367 return NULL;
368
369error:
370 if (fd) {
371 close(fd);
372 }
373 /* Don't dead lock in main during initialization */
374 bsem_post(data->sem);
375 return NULL;
376}
377
378/* High resolution timer */
379static double gettime()
380{
381 struct timeval t;
382 struct timezone tzp;
383 gettimeofday(&t, &tzp);
384 return t.tv_sec + t.tv_usec*1e-6;
385}
386
387/* Human readable size metric from size, not thread safe */
388static const char* sizemetric(size_t size)
389{
390 static const char *sizes[] = { "B", "KiB", "MiB", "GiB", "TiB" };
391
392 /* Find the suffix */
393 double bytes = (float)size;
394 size_t index = 0;
395 for (; bytes >= 1024.0 && index < sizeof sizes / sizeof *sizes; index++ )
396 {
397 bytes /= 1024.0;
398 }
399
400 assert(index != sizeof sizes / sizeof *sizes);
401
402 /* Truncate the representation if needed */
403 char buffer[2*(DBL_MANT_DIG+DBL_MAX_EXP)];
404 const int ret = snprintf(buffer, sizeof buffer, "%.*f", (int)sizeof buffer, bytes ); //-V512 truncation expected
405 assert(ret > 0);
406 /* Remove everything after (including) the period */
407 char *period = strchr(buffer, '.');
408 assert(period);
409 period[3] = '\0';
410
411 /* Format with human readable suffix */
412 static char format[sizeof buffer + 5];
413 snprintf(format, sizeof format, "%s %s", buffer, sizes[index] );
414 return format;
415}
416
417static int enumeratecpus(void) {
418 FILE* fp = popen("lscpu -p", "r");
419 if (!fp) {
420 return -1;
421 }
422
423 int maxcpus = 0;
424 int maxcores = 0;
425 char* linebuf = NULL;
426 size_t linelen = 0;
427 while (getline(&linebuf, &linelen, fp) != EOF) {
428 if (*linebuf == '#') {
429 continue;
430 }
431 int cpus = 0;
432 int cores = 0;
433 if (sscanf(linebuf, "%d,%d", &cpus, &cores) == 2) {
434 if (cpus > maxcpus) {
435 maxcpus = cpus;
436 }
437 if (cores > maxcores) {
438 maxcores = cores;
439 }
440 }
441 }
442 free(linebuf);
443
444 printf("discovered %d logcial cpus, %d physical, %d threads per core\n", maxcpus+1, maxcores+1, maxcpus/maxcores);
445 fclose(fp);
446 return maxcpus/maxcores;
447}
448
449static void usage(const char *app, FILE *fp) {
450 fprintf(fp, "usage: %s [options]\n", app);
451 fprintf(fp, "options:\n"
452 " -s, --same force read and write on cpu\n"
453 " -h, --help print this help message\n"
454 " -m, --memory=MB the amount of memory to work on in MB\n"
455 " -t, --trials=COUNT the amount of trials to run for benchmark\n");
456}
457
458static int isparam(int argc, char **argv, int *arg, char sh, const char *lng, char **argarg) {
459 if (argv[*arg][0] != '-') {
460 return 0;
461 }
462 /* short version */
463 if (argv[*arg][1] == sh) {
464 if (argv[*arg][2]) {
465 *argarg = argv[*arg]+2;
466 return 1;
467 }
468 ++*arg;
469 if (*arg == argc) {
470 fprintf(stderr, "%s: option -%c requires an argument\n", argv[0], sh);
471 usage(argv[0], stderr);
472 *arg = -1;
473 return 1;
474 }
475 *argarg = argv[*arg];
476 return 1;
477 }
478
479 /* long version */
480 if (argv[*arg][1] != '-') {
481 return 0;
482 }
483 size_t len = strlen(lng);
484 if (strncmp(argv[*arg]+2, lng, len)) {
485 return 0;
486 }
487 if (argv[*arg][len+2] == '=') {
488 *argarg = argv[*arg] + 3 + len;
489 return 1;
490 }
491 if (!argv[*arg][len+2]) {
492 ++*arg;
493 if (*arg == argc) {
494 fprintf(stderr, "%s: option --%s requires an argument\n", argv[0], lng);
495 usage(argv[0], stderr);
496 *arg = -1;
497 return 1;
498 }
499 *argarg = argv[*arg];
500 return 1;
501 }
502 return 0;
503}
504
505int main(int argc, char **argv)
506{
507 int trials = TRIALS;
508 size_t memory = MEMORY;
509 int same = 0;
510
511 int arg = 1;
512 for (; arg != argc; ++arg) {
513 char *argarg = NULL;
514 if (!strcmp(argv[arg], "-h") || !strcmp(argv[arg], "--help")) {
515 usage(argv[0], stdout);
516 return EXIT_SUCCESS;
517 }
518 if (!strcmp(argv[arg], "-s") || !strcmp(argv[arg], "--same")) {
519 same = 1;
520 continue;
521 }
522 if (isparam(argc, argv, &arg, 't', "trials", &argarg)) {
523 if (arg < 0) {
524 return EXIT_FAILURE;
525 }
526 trials = atoi(argarg);
527 continue;
528 }
529 if (isparam(argc, argv, &arg, 'm', "memory", &argarg)) {
530 if (arg < 0) {
531 return EXIT_FAILURE;
532 }
533 memory = (size_t)atoi(argarg) * 1024ull * 1024ull;
534 continue;
535 }
536 fprintf(stderr, "unknown option: %s\n", argv[arg]);
537 return EXIT_FAILURE;
538 }
539
540 int threads = enumeratecpus();
541
542 printf("measuing memory perf across CPUs with explicit memory mappings\n");
543 printf("running %zu trials on a space of %s\n", trials, sizemetric(memory));
544
545 /* Create shared memory for each thread to utilize. We cannot use
546 * this processes memory because this process may be on the same
547 * core as the read or write thread. We want each thread to get its
548 * own mapping for it.
549 */
550 int fd = shm_open("/xcpumemperf", O_CREAT | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR);
551 if (fd < 0) {
552 err("failed to create shared memory\n");
553 return EXIT_FAILURE;
554 }
555 /* No longer need the name */
556 shm_unlink("/xcpumemperf");
557 if (ftruncate(fd, memory) < 0) {
558 err("failed to truncate shared memory\n");
559 close(fd);
560 return EXIT_FAILURE;
561 }
562
563 unsigned char* touch = mmap(0, memory, PROT_NONE, MAP_SHARED | MAP_POPULATE, fd, 0);
564 if (touch == MAP_FAILED) {
565 err("failed to map shared memory\n");
566 close(fd);
567 return EXIT_FAILURE;
568 }
569
570 /* Create a socket to communicate the shared memory file descriptor
571 * on, each thread will open the shared memory with its own mapping
572 * this way such that reads and writes behave through their own
573 * translation and not through the same virtual address this process
574 * gets.
575 */
576 int pair[2];
577 if (socketpair(PF_LOCAL, SOCK_STREAM, 0, pair) < 0) {
578 err("failed to create socket pair for communicating\n");
579 return EXIT_FAILURE;
580 }
581
582 pthread_t rdthr;
583 struct bsem rdsem;
584 pthread_t wrthr;
585 struct bsem wrsem;
586
587 if (bsem_init(&rdsem) < 0 || bsem_init(&wrsem) < 0) {
588 err("failed to create semaphores for communicating\n");
589 close(pair[0]);
590 close(pair[1]);
591 return EXIT_FAILURE;
592 }
593
594 struct threaddata rdthrd = { .threads = same ? 0 : threads, .memory = memory, .sem = &rdsem, .result = -1, .socket = pair[1] };
595 struct threaddata wrthrd = { .threads = same ? 0 : threads, .memory = memory, .sem = &wrsem, .result = -1, .socket = pair[1] };
596
597 /* Run trials */
598 double tbeg = gettime();
599 double rdtime = 0.0;
600 double wrtime = 0.0;
601 for (size_t i = 0; i < trials; i++)
602 {
603 double wrbeg = gettime();
604 /* Create write thred */
605 pthread_create(&wrthr, NULL, wr, &wrthrd);
606 /* Write the shared memory file descriptor to the pipe */
607 sendfd(pair[0], fd);
608 /* Wait for the write thread to start */
609 bsem_wait(&wrsem);
610
611 double rdbeg = gettime();
612 /* Create read thread */
613 pthread_create(&rdthr, NULL, rd, &rdthrd);
614 /* Write the shared memory file descriptor to the pipe */
615 sendfd(pair[0], fd);
616 /* Wait for the read thread to start */
617 bsem_wait(&rdsem);
618
619 /* Wait for write and read threads to complete */
620 pthread_join(wrthr, NULL);
621 double wrend = gettime();
622
623 pthread_join(rdthr, NULL);
624 double rdend = gettime();
625
626 /* Calculate differences */
627 double wrdif = wrend - wrbeg;
628 double rddif = rdend - rdbeg;
629 wrtime += wrdif;
630 rdtime += rddif;
631
632 printf("\rtrial %zu of %zu [%%% 3.2f] (wr %f sec, rd %f sec)", i+1, trials, (float)(i+1)/(float)trials*100.0f, wrdif, rddif);
633 fflush(stdout);
634 }
635 printf("\n");
636 double tend = gettime();
637
638 bsem_destroy(&wrsem);
639 bsem_destroy(&rdsem);
640
641 /* Unmap shared memory */
642 munmap(touch, memory);
643
644 /* Close socket pair */
645 close(pair[0]);
646 close(pair[1]);
647
648 if (rdthrd.result < memory - 1 || wrthrd.result < memory - 1) {
649 err("incomplete results\n");
650 return EXIT_FAILURE;
651 }
652
653 printf("finished average: (wr %f sec, rd %f sec) benchmark took %f secs total\n", wrtime/trials, rdtime/trials, tend-tbeg);
654 return EXIT_SUCCESS;
655}