nbd-tester-client.c

   1 /*
   2  * Test client to test the NBD server. Doesn't do anything useful, except
   3  * checking that the server does, actually, work.
   4  *
   5  * Note that the only 'real' test is to check the client against a kernel. If
   6  * it works here but does not work in the kernel, then that's most likely a bug
   7  * in this program and/or in nbd-server.
   8  *
   9  * Copyright(c) 2006  Wouter Verhelst
  10  *
  11  * This program is Free Software; you can redistribute it and/or modify it
  12  * under the terms of the GNU General Public License as published by the Free
  13  * Software Foundation, in version 2.
  14  *
  15  * This program is distributed in the hope that it will be useful, but WITHOUT
  16  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  17  * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  18  * more details.
  19  *
  20  * You should have received a copy of the GNU General Public License along with
  21  * this program; if not, write to the Free Software Foundation, Inc., 51
  22  * Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  23  */
  24 #include <stdlib.h>
  25 #include <stdio.h>
  26 #include <stdbool.h>
  27 #include <string.h>
  28 #include <sys/time.h>
  29 #include <sys/types.h>
  30 #include <sys/socket.h>
  31 #include <sys/stat.h>
  32 #include <sys/mman.h>
  33 #include <fcntl.h>
  34 #include <syslog.h>
  35 #include <unistd.h>
  36 #include "config.h"
  37 #include "lfs.h"
  38 #define MY_NAME "nbd-tester-client"
  39 #include "cliserv.h"
  40
  41 #include <netinet/in.h>
  42 #include <glib.h>
  43
  44 static gchar errstr[1024];
  45 const static int errstr_len=1024;
  46
  47 static uint64_t size;
  48
  49 static gchar * transactionlog = "nbd-tester-client.tr";
  50
  51 typedef enum {
  52         CONNECTION_TYPE_NONE,
  53         CONNECTION_TYPE_CONNECT,
  54         CONNECTION_TYPE_INIT_PASSWD,
  55         CONNECTION_TYPE_CLISERV,
  56         CONNECTION_TYPE_FULL,
  57 } CONNECTION_TYPE;
  58
  59 typedef enum {
  60         CONNECTION_CLOSE_PROPERLY,
  61         CONNECTION_CLOSE_FAST,
  62 } CLOSE_TYPE;
  63
  64 struct reqcontext {
  65         uint64_t seq;
  66         struct nbd_request req;
  67         struct reqcontext * next;
  68         struct reqcontext * prev;
  69 };
  70
  71 struct rclist {
  72         struct reqcontext * head;
  73         struct reqcontext * tail;
  74         int numitems;
  75 };
  76
  77 void rclist_unlink(struct rclist * l, struct reqcontext * p) {
  78         if (p && l) {
  79                 struct reqcontext * prev = p->prev;
  80                 struct reqcontext * next = p->next;
  81
  82                 /* Fix link to previous */
  83                 if (prev)
  84                         prev->next = next;
  85                 else
  86                         l->head = next;
  87
  88                 if (next)
  89                         next->prev = prev;
  90                 else
  91                         l->tail = prev;
  92
  93                 p->prev = NULL;
  94                 p->next = NULL;
  95                 l->numitems--;
  96         }
  97 }
  98
  99 /* Add a new list item to the tail */
 100 void rclist_addtail(struct rclist * l, struct reqcontext * p)
 101 {
 102         if (!p || !l)
 103                 return;
 104         if (l->tail) {
 105                 if (l->tail->next)
 106                         g_warning("addtail found list tail has a next pointer");
 107                 l->tail->next = p;
 108                 p->next = NULL;
 109                 p->prev = l->tail;
 110                 l->tail = p;
 111         } else {
 112                 if (l->head)
 113                         g_warning("addtail found no list tail but a list head");
 114                 l->head = p;
 115                 l->tail = p;
 116                 p->prev = NULL;
 117                 p->next = NULL;
 118         }
 119         l->numitems++;
 120 }
 121
 122 #define TEST_WRITE (1<<0)
 123 #define TEST_FLUSH (1<<1)
 124
 125 int timeval_subtract (struct timeval *result, struct timeval *x,
 126                       struct timeval *y) {
 127         if (x->tv_usec < y->tv_usec) {
 128                 int nsec = (y->tv_usec - x->tv_usec) / 1000000 + 1;
 129                 y->tv_usec -= 1000000 * nsec;
 130                 y->tv_sec += nsec;
 131         }
 132
 133         if (x->tv_usec - y->tv_usec > 1000000) {
 134                 int nsec = (x->tv_usec - y->tv_usec) / 1000000;
 135                 y->tv_usec += 1000000 * nsec;
 136                 y->tv_sec -= nsec;
 137         }
 138
 139         result->tv_sec = x->tv_sec - y->tv_sec;
 140         result->tv_usec = x->tv_usec - y->tv_usec;
 141
 142         return x->tv_sec < y->tv_sec;
 143 }
 144
 145 double timeval_diff_to_double (struct timeval * x, struct timeval * y) {
 146         struct timeval r;
 147         timeval_subtract(&r, x, y);
 148         return r.tv_sec * 1.0 + r.tv_usec/1000000.0;
 149 }
 150
 151 static inline int read_all(int f, void *buf, size_t len) {
 152         ssize_t res;
 153         size_t retval=0;
 154
 155         while(len>0) {
 156                 if((res=read(f, buf, len)) <=0) {
 157                         if (!res)
 158                                 errno=EAGAIN;
 159                         snprintf(errstr, errstr_len, "Read failed: %s", strerror(errno));
 160                         return -1;
 161                 }
 162                 len-=res;
 163                 buf+=res;
 164                 retval+=res;
 165         }
 166         return retval;
 167 }
 168
 169 static inline int write_all(int f, void *buf, size_t len) {
 170         ssize_t res;
 171         size_t retval=0;
 172
 173         while(len>0) {
 174                 if((res=write(f, buf, len)) <=0) {
 175                         if (!res)
 176                                 errno=EAGAIN;
 177                         snprintf(errstr, errstr_len, "Write failed: %s", strerror(errno));
 178                         return -1;
 179                 }
 180                 len-=res;
 181                 buf+=res;
 182                 retval+=res;
 183         }
 184         return retval;
 185 }
 186
 187 #define READ_ALL_ERRCHK(f, buf, len, whereto, errmsg...) if((read_all(f, buf, len))<=0) { snprintf(errstr, errstr_len, ##errmsg); goto whereto; }
 188 #define READ_ALL_ERR_RT(f, buf, len, whereto, rval, errmsg...) if((read_all(f, buf, len))<=0) { snprintf(errstr, errstr_len, ##errmsg); retval = rval; goto whereto; }
 189
 190 #define WRITE_ALL_ERRCHK(f, buf, len, whereto, errmsg...) if((write_all(f, buf, len))<=0) { snprintf(errstr, errstr_len, ##errmsg); goto whereto; }
 191 #define WRITE_ALL_ERR_RT(f, buf, len, whereto, rval, errmsg...) if((write_all(f, buf, len))<=0) { snprintf(errstr, errstr_len, ##errmsg); retval = rval; goto whereto; }
 192
 193 int setup_connection(gchar *hostname, int port, gchar* name, CONNECTION_TYPE ctype, int* serverflags) {
 194         int sock;
 195         struct hostent *host;
 196         struct sockaddr_in addr;
 197         char buf[256];
 198         uint64_t mymagic = (name ? opts_magic : cliserv_magic);
 199         u64 tmp64;
 200         uint32_t tmp32 = 0;
 201
 202         sock=0;
 203         if(ctype<CONNECTION_TYPE_CONNECT)
 204                 goto end;
 205         if((sock=socket(PF_INET, SOCK_STREAM, IPPROTO_TCP))<0) {
 206                 strncpy(errstr, strerror(errno), errstr_len);
 207                 goto err;
 208         }
 209         setmysockopt(sock);
 210         if(!(host=gethostbyname(hostname))) {
 211                 strncpy(errstr, strerror(errno), errstr_len);
 212                 goto err_open;
 213         }
 214         addr.sin_family=AF_INET;
 215         addr.sin_port=htons(port);
 216         addr.sin_addr.s_addr=*((int *) host->h_addr);
 217         if((connect(sock, (struct sockaddr *)&addr, sizeof(addr))<0)) {
 218                 strncpy(errstr, strerror(errno), errstr_len);
 219                 goto err_open;
 220         }
 221         if(ctype<CONNECTION_TYPE_INIT_PASSWD)
 222                 goto end;
 223         READ_ALL_ERRCHK(sock, buf, strlen(INIT_PASSWD), err_open, "Could not read INIT_PASSWD: %s", strerror(errno));
 224         if(strlen(buf)==0) {
 225                 snprintf(errstr, errstr_len, "Server closed connection");
 226                 goto err_open;
 227         }
 228         if(strncmp(buf, INIT_PASSWD, strlen(INIT_PASSWD))) {
 229                 snprintf(errstr, errstr_len, "INIT_PASSWD does not match");
 230                 goto err_open;
 231         }
 232         if(ctype<CONNECTION_TYPE_CLISERV)
 233                 goto end;
 234         READ_ALL_ERRCHK(sock, &tmp64, sizeof(tmp64), err_open, "Could not read cliserv_magic: %s", strerror(errno));
 235         tmp64=ntohll(tmp64);
 236         if(tmp64 != mymagic) {
 237                 strncpy(errstr, "mymagic does not match", errstr_len);
 238                 goto err_open;
 239         }
 240         if(ctype<CONNECTION_TYPE_FULL)
 241                 goto end;
 242         if(!name) {
 243                 READ_ALL_ERRCHK(sock, &size, sizeof(size), err_open, "Could not read size: %s", strerror(errno));
 244                 size=ntohll(size);
 245                 READ_ALL_ERRCHK(sock, buf, 128, err_open, "Could not read data: %s", strerror(errno));
 246                 goto end;
 247         }
 248         /* flags */
 249         READ_ALL_ERRCHK(sock, buf, sizeof(uint16_t), err_open, "Could not read reserved field: %s", strerror(errno));
 250         /* reserved field */
 251         WRITE_ALL_ERRCHK(sock, &tmp32, sizeof(tmp32), err_open, "Could not write reserved field: %s", strerror(errno));
 252         /* magic */
 253         tmp64 = htonll(opts_magic);
 254         WRITE_ALL_ERRCHK(sock, &tmp64, sizeof(tmp64), err_open, "Could not write magic: %s", strerror(errno));
 255         /* name */
 256         tmp32 = htonl(NBD_OPT_EXPORT_NAME);
 257         WRITE_ALL_ERRCHK(sock, &tmp32, sizeof(tmp32), err_open, "Could not write option: %s", strerror(errno));
 258         tmp32 = htonl((uint32_t)strlen(name));
 259         WRITE_ALL_ERRCHK(sock, &tmp32, sizeof(tmp32), err_open, "Could not write name length: %s", strerror(errno));
 260         WRITE_ALL_ERRCHK(sock, name, strlen(name), err_open, "Could not write name:: %s", strerror(errno));
 261         READ_ALL_ERRCHK(sock, &size, sizeof(size), err_open, "Could not read size: %s", strerror(errno));
 262         size = ntohll(size);
 263         uint16_t flags;
 264         READ_ALL_ERRCHK(sock, &flags, sizeof(uint16_t), err_open, "Could not read flags: %s", strerror(errno));
 265         flags = ntohs(flags);
 266         *serverflags = flags;
 267         READ_ALL_ERRCHK(sock, buf, 124, err_open, "Could not read reserved zeroes: %s", strerror(errno));
 268         goto end;
 269 err_open:
 270         close(sock);
 271 err:
 272         sock=-1;
 273 end:
 274         return sock;
 275 }
 276
 277 int close_connection(int sock, CLOSE_TYPE type) {
 278         struct nbd_request req;
 279         u64 counter=0;
 280
 281         switch(type) {
 282                 case CONNECTION_CLOSE_PROPERLY:
 283                         req.magic=htonl(NBD_REQUEST_MAGIC);
 284                         req.type=htonl(NBD_CMD_DISC);
 285                         memcpy(&(req.handle), &(counter), sizeof(counter));
 286                         counter++;
 287                         req.from=0;
 288                         req.len=0;
 289                         if(write(sock, &req, sizeof(req))<0) {
 290                                 snprintf(errstr, errstr_len, "Could not write to socket: %s", strerror(errno));
 291                                 return -1;
 292                         }
 293                 case CONNECTION_CLOSE_FAST:
 294                         if(close(sock)<0) {
 295                                 snprintf(errstr, errstr_len, "Could not close socket: %s", strerror(errno));
 296                                 return -1;
 297                         }
 298                         break;
 299                 default:
 300                         g_critical("Your compiler is on crack!"); /* or I am buggy */
 301                         return -1;
 302         }
 303         return 0;
 304 }
 305
 306 int read_packet_check_header(int sock, size_t datasize, long long int curhandle) {
 307         struct nbd_reply rep;
 308         int retval=0;
 309         char buf[datasize];
 310
 311         READ_ALL_ERR_RT(sock, &rep, sizeof(rep), end, -1, "Could not read reply header: %s", strerror(errno));
 312         rep.magic=ntohl(rep.magic);
 313         rep.error=ntohl(rep.error);
 314         if(rep.magic!=NBD_REPLY_MAGIC) {
 315                 snprintf(errstr, errstr_len, "Received package with incorrect reply_magic. Index of sent packages is %lld (0x%llX), received handle is %lld (0x%llX). Received magic 0x%lX, expected 0x%lX", (long long int)curhandle, (long long unsigned int)curhandle, (long long int)*((u64*)rep.handle), (long long unsigned int)*((u64*)rep.handle), (long unsigned int)rep.magic, (long unsigned int)NBD_REPLY_MAGIC);
 316                 retval=-1;
 317                 goto end;
 318         }
 319         if(rep.error) {
 320                 snprintf(errstr, errstr_len, "Received error from server: %ld (0x%lX). Handle is %lld (0x%llX).", (long int)rep.error, (long unsigned int)rep.error, (long long int)(*((u64*)rep.handle)), (long long unsigned int)*((u64*)rep.handle));
 321                 retval=-1;
 322                 goto end;
 323         }
 324         if (datasize)
 325                 READ_ALL_ERR_RT(sock, &buf, datasize, end, -1, "Could not read data: %s", strerror(errno));
 326
 327 end:
 328         return retval;
 329 }
 330
 331 int oversize_test(gchar* hostname, int port, char* name, int sock,
 332                   char sock_is_open, char close_sock, int testflags) {
 333         int retval=0;
 334         struct nbd_request req;
 335         struct nbd_reply rep;
 336         int request=0;
 337         int i=0;
 338         int serverflags = 0;
 339         pid_t mypid = getpid();
 340         char buf[((1024*1024)+sizeof(struct nbd_request)/2)<<1];
 341         bool got_err;
 342
 343         /* This should work */
 344         if(!sock_is_open) {
 345                 if((sock=setup_connection(hostname, port, name, CONNECTION_TYPE_FULL, &serverflags))<0) {
 346                         g_warning("Could not open socket: %s", errstr);
 347                         retval=-1;
 348                         goto err;
 349                 }
 350         }
 351         req.magic=htonl(NBD_REQUEST_MAGIC);
 352         req.type=htonl(NBD_CMD_READ);
 353         req.len=htonl(1024*1024);
 354         memcpy(&(req.handle),&i,sizeof(i));
 355         req.from=htonll(i);
 356         WRITE_ALL_ERR_RT(sock, &req, sizeof(req), err, -1, "Could not write request: %s", strerror(errno));
 357         printf("%d: testing oversized request: %d: ", getpid(), ntohl(req.len));
 358         READ_ALL_ERR_RT(sock, &rep, sizeof(struct nbd_reply), err, -1, "Could not read reply header: %s", strerror(errno));
 359         READ_ALL_ERR_RT(sock, &buf, ntohl(req.len), err, -1, "Could not read data: %s", strerror(errno));
 360         if(rep.error) {
 361                 snprintf(errstr, errstr_len, "Received unexpected error: %d", rep.error);
 362                 retval=-1;
 363                 goto err;
 364         } else {
 365                 printf("OK\n");
 366         }
 367         /* This probably should not work */
 368         i++; req.from=htonll(i);
 369         req.len = htonl(ntohl(req.len) + sizeof(struct nbd_request) / 2);
 370         WRITE_ALL_ERR_RT(sock, &req, sizeof(req), err, -1, "Could not write request: %s", strerror(errno));
 371         printf("%d: testing oversized request: %d: ", getpid(), ntohl(req.len));
 372         READ_ALL_ERR_RT(sock, &rep, sizeof(struct nbd_reply), err, -1, "Could not read reply header: %s", strerror(errno));
 373         READ_ALL_ERR_RT(sock, &buf, ntohl(req.len), err, -1, "Could not read data: %s", strerror(errno));
 374         if(rep.error) {
 375                 printf("Received expected error\n");
 376                 got_err=true;
 377         } else {
 378                 printf("OK\n");
 379                 got_err=false;
 380         }
 381         /* ... unless this works, too */
 382         i++; req.from=htonll(i);
 383         req.len = htonl(ntohl(req.len) << 1);
 384         WRITE_ALL_ERR_RT(sock, &req, sizeof(req), err, -1, "Could not write request: %s", strerror(errno));
 385         printf("%d: testing oversized request: %d: ", getpid(), ntohl(req.len));
 386         READ_ALL_ERR_RT(sock, &rep, sizeof(struct nbd_reply), err, -1, "Could not read reply header: %s", strerror(errno));
 387         READ_ALL_ERR_RT(sock, &buf, ntohl(req.len), err, -1, "Could not read data: %s", strerror(errno));
 388         if(rep.error) {
 389                 printf("error\n");
 390         } else {
 391                 printf("OK\n");
 392         }
 393         if((rep.error && !got_err) || (!rep.error && got_err)) {
 394                 printf("Received unexpected error\n");
 395                 retval=-1;
 396         }
 397   err:
 398         return retval;
 399 }
 400
 401 int throughput_test(gchar* hostname, int port, char* name, int sock,
 402                     char sock_is_open, char close_sock, int testflags) {
 403         long long int i;
 404         char buf[1024];
 405         char writebuf[1024];
 406         struct nbd_request req;
 407         int requests=0;
 408         fd_set set;
 409         struct timeval tv;
 410         struct timeval start;
 411         struct timeval stop;
 412         double timespan;
 413         double speed;
 414         char speedchar[2] = { '\0', '\0' };
 415         int retval=0;
 416         int serverflags = 0;
 417         size_t tmp;
 418         signed int do_write=TRUE;
 419         pid_t mypid = getpid();
 420
 421
 422         if (!(testflags & TEST_WRITE))
 423                 testflags &= ~TEST_FLUSH;
 424
 425         memset (writebuf, 'X', 1024);
 426         size=0;
 427         if(!sock_is_open) {
 428                 if((sock=setup_connection(hostname, port, name, CONNECTION_TYPE_FULL, &serverflags))<0) {
 429                         g_warning("Could not open socket: %s", errstr);
 430                         retval=-1;
 431                         goto err;
 432                 }
 433         }
 434         if ((testflags & TEST_FLUSH) && ((serverflags & (NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA))
 435                                          != (NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA))) {
 436                 snprintf(errstr, errstr_len, "Server did not supply flush capability flags");
 437                 retval = -1;
 438                 goto err_open;
 439         }
 440         req.magic=htonl(NBD_REQUEST_MAGIC);
 441         req.len=htonl(1024);
 442         if(gettimeofday(&start, NULL)<0) {
 443                 retval=-1;
 444                 snprintf(errstr, errstr_len, "Could not measure start time: %s", strerror(errno));
 445                 goto err_open;
 446         }
 447         for(i=0;i+1024<=size;i+=1024) {
 448                 if(do_write) {
 449                         int sendfua = (testflags & TEST_FLUSH) && (((i>>10) & 15) == 3);
 450                         int sendflush = (testflags & TEST_FLUSH) && (((i>>10) & 15) == 11);
 451                         req.type=htonl((testflags & TEST_WRITE)?NBD_CMD_WRITE:NBD_CMD_READ);
 452                         if (sendfua)
 453                                 req.type = htonl(NBD_CMD_WRITE | NBD_CMD_FLAG_FUA);
 454                         memcpy(&(req.handle),&i,sizeof(i));
 455                         req.from=htonll(i);
 456                         if (write_all(sock, &req, sizeof(req)) <0) {
 457                                 retval=-1;
 458                                 goto err_open;
 459                         }
 460                         if (testflags & TEST_WRITE) {
 461                                 if (write_all(sock, writebuf, 1024) <0) {
 462                                         retval=-1;
 463                                         goto err_open;
 464                                 }
 465                         }
 466                         printf("%d: Requests(+): %d\n", (int)mypid, ++requests);
 467                         if (sendflush) {
 468                                 long long int j = i ^ (1LL<<63);
 469                                 req.type = htonl(NBD_CMD_FLUSH);
 470                                 memcpy(&(req.handle),&j,sizeof(j));
 471                                 req.from=0;
 472                                 if (write_all(sock, &req, sizeof(req)) <0) {
 473                                         retval=-1;
 474                                         goto err_open;
 475                                 }
 476                                 printf("%d: Requests(+): %d\n", (int)mypid, ++requests);
 477                         }
 478                 }
 479                 do {
 480                         FD_ZERO(&set);
 481                         FD_SET(sock, &set);
 482                         tv.tv_sec=0;
 483                         tv.tv_usec=0;
 484                         select(sock+1, &set, NULL, NULL, &tv);
 485                         if(FD_ISSET(sock, &set)) {
 486                                 /* Okay, there's something ready for
 487                                  * reading here */
 488                                 if(read_packet_check_header(sock, (testflags & TEST_WRITE)?0:1024, i)<0) {
 489                                         retval=-1;
 490                                         goto err_open;
 491                                 }
 492                                 printf("%d: Requests(-): %d\n", (int)mypid, --requests);
 493                         }
 494                 } while FD_ISSET(sock, &set);
 495                 /* Now wait until we can write again or until a second have
 496                  * passed, whichever comes first*/
 497                 FD_ZERO(&set);
 498                 FD_SET(sock, &set);
 499                 tv.tv_sec=1;
 500                 tv.tv_usec=0;
 501                 do_write=select(sock+1,NULL,&set,NULL,&tv);
 502                 if(!do_write) printf("Select finished\n");
 503                 if(do_write<0) {
 504                         snprintf(errstr, errstr_len, "select: %s", strerror(errno));
 505                         retval=-1;
 506                         goto err_open;
 507                 }
 508         }
 509         /* Now empty the read buffer */
 510         do {
 511                 FD_ZERO(&set);
 512                 FD_SET(sock, &set);
 513                 tv.tv_sec=0;
 514                 tv.tv_usec=0;
 515                 select(sock+1, &set, NULL, NULL, &tv);
 516                 if(FD_ISSET(sock, &set)) {
 517                         /* Okay, there's something ready for
 518                          * reading here */
 519                         read_packet_check_header(sock, (testflags & TEST_WRITE)?0:1024, i);
 520                         printf("%d: Requests(-): %d\n", (int)mypid, --requests);
 521                 }
 522         } while (requests);
 523         if(gettimeofday(&stop, NULL)<0) {
 524                 retval=-1;
 525                 snprintf(errstr, errstr_len, "Could not measure end time: %s", strerror(errno));
 526                 goto err_open;
 527         }
 528         timespan=timeval_diff_to_double(&stop, &start);
 529         speed=size/timespan;
 530         if(speed>1024) {
 531                 speed=speed/1024.0;
 532                 speedchar[0]='K';
 533         }
 534         if(speed>1024) {
 535                 speed=speed/1024.0;
 536                 speedchar[0]='M';
 537         }
 538         if(speed>1024) {
 539                 speed=speed/1024.0;
 540                 speedchar[0]='G';
 541         }
 542         g_message("%d: Throughput %s test (%s flushes) complete. Took %.3f seconds to complete, %.3f%sib/s", (int)getpid(), (testflags & TEST_WRITE)?"write":"read", (testflags & TEST_FLUSH)?"with":"without", timespan, speed, speedchar);
 543
 544 err_open:
 545         if(close_sock) {
 546                 close_connection(sock, CONNECTION_CLOSE_PROPERLY);
 547         }
 548 err:
 549         return retval;
 550 }
 551
 552 /*
 553  * fill 512 byte buffer 'buf' with a hashed selection of interesting data based
 554  * only on handle and blknum. The first word is blknum, and the second handle, for ease
 555  * of understanding. Things with handle 0 are blank.
 556  */
 557 static inline void makebuf(char *buf, uint64_t seq, uint64_t blknum) {
 558         uint64_t x = ((uint64_t)blknum) ^ (seq << 32) ^ (seq >> 32);
 559         uint64_t* p = (uint64_t*)buf;
 560         int i;
 561         if (!seq) {
 562                 bzero(buf, 512);
 563                 return;
 564         }
 565         for (i = 0; i<512/sizeof(uint64_t); i++) {
 566                 int s;
 567                 *(p++) = x;
 568                 x+=0xFEEDA1ECDEADBEEFULL+i+(((uint64_t)i)<<56);
 569                 s = x & 63;
 570                 x = x ^ (x<<s) ^ (x>>(64-s)) ^ 0xAA55AA55AA55AA55ULL ^ seq;
 571         }
 572 }
 573
 574 static inline int checkbuf(char *buf, uint64_t seq, uint64_t blknum) {
 575         char cmp[512];
 576         makebuf(cmp, seq, blknum);
 577         return memcmp(cmp, buf, 512)?-1:0;
 578 }
 579
 580 static inline void dumpcommand(char * text, uint32_t command)
 581 {
 582 #ifdef DEBUG_COMMANDS
 583         command=ntohl(command);
 584         char * ctext;
 585         switch (command & NBD_CMD_MASK_COMMAND) {
 586         case NBD_CMD_READ:
 587                 ctext="NBD_CMD_READ";
 588                 break;
 589         case NBD_CMD_WRITE:
 590                 ctext="NBD_CMD_WRITE";
 591                 break;
 592         case NBD_CMD_DISC:
 593                 ctext="NBD_CMD_DISC";
 594                 break;
 595         case NBD_CMD_FLUSH:
 596                 ctext="NBD_CMD_FLUSH";
 597                 break;
 598         default:
 599                 ctext="UNKNOWN";
 600                 break;
 601         }
 602         printf("%s: %s [%s] (0x%08x)\n",
 603                text,
 604                ctext,
 605                (command & NBD_CMD_FLAG_FUA)?"FUA":"NONE",
 606                command);
 607 #endif
 608 }
 609
 610 int integrity_test(gchar* hostname, int port, char* name, int sock,
 611                    char sock_is_open, char close_sock, int testflags) {
 612         struct nbd_request req;
 613         struct nbd_reply rep;
 614         fd_set rset;
 615         fd_set wset;
 616         struct timeval tv;
 617         struct timeval start;
 618         struct timeval stop;
 619         double timespan;
 620         double speed;
 621         char speedchar[2] = { '\0', '\0' };
 622         int retval=0;
 623         int serverflags = 0;
 624         pid_t mypid = getpid();
 625         int blkhashfd = -1;
 626         char *blkhashname=NULL;
 627         uint32_t *blkhash = NULL;
 628         int logfd=-1;
 629         uint64_t seq=1;
 630         uint64_t processed=0;
 631         uint64_t printer=0;
 632         int readtransactionfile = 1;
 633         struct rclist txqueue={NULL, NULL, 0};
 634         struct rclist inflight={NULL, NULL, 0};
 635
 636         size=0;
 637         if(!sock_is_open) {
 638                 if((sock=setup_connection(hostname, port, name, CONNECTION_TYPE_FULL, &serverflags))<0) {
 639                         g_warning("Could not open socket: %s", errstr);
 640                         retval=-1;
 641                         goto err;
 642                 }
 643         }
 644
 645         if ((serverflags & (NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA))
 646             != (NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA))
 647                 g_warning("Server flags do not support FLUSH and FUA - these may error");
 648
 649 #ifdef HAVE_MKSTEMP
 650         blkhashname=strdup("/tmp/blkarray-XXXXXX");
 651         if (!blkhashname || (-1 == (blkhashfd = mkstemp(blkhashname)))) {
 652                 g_warning("Could not open temp file: %s", strerror(errno));
 653                 retval=-1;
 654                 goto err;
 655         }
 656 #else
 657         /* use tmpnam here to avoid further feature test nightmare */
 658         if (-1 == (blkhashfd = open(blkhashname=strdup(tmpnam(NULL)),
 659                                     O_CREAT | O_RDWR,
 660                                     S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH))) {
 661                 g_warning("Could not open temp file: %s", strerror(errno));
 662                 retval=-1;
 663                 goto err;
 664         }
 665 #endif
 666         /* Ensure space freed if we die */
 667         if (-1 == unlink(blkhashname)) {
 668                 g_warning("Could not unlink temp file: %s", strerror(errno));
 669                 retval=-1;
 670                 goto err;
 671         }
 672
 673         if (-1 == lseek(blkhashfd, (off_t)((size>>9)<<2), SEEK_SET)) {
 674                 g_warning("Could not llseek temp file: %s", strerror(errno));
 675                 retval=-1;
 676                 goto err;
 677         }
 678
 679         if (-1 == write(blkhashfd, "\0", 1)) {
 680                 g_warning("Could not write temp file: %s", strerror(errno));
 681                 retval=-1;
 682                 goto err;
 683         }
 684
 685         if (NULL == (blkhash = mmap(NULL,
 686                                     (size>>9)<<2,
 687                                     PROT_READ | PROT_WRITE,
 688                                     MAP_SHARED,
 689                                     blkhashfd,
 690                                     0))) {
 691                 g_warning("Could not mmap temp file: %s", strerror(errno));
 692                 retval=-1;
 693                 goto err;
 694         }
 695
 696         if (-1 == (logfd = open(transactionlog, O_RDONLY)))
 697         {
 698                 g_warning("Could open log file: %s", strerror(errno));
 699                 retval=-1;
 700                 goto err;
 701         }
 702
 703         if(gettimeofday(&start, NULL)<0) {
 704                 retval=-1;
 705                 snprintf(errstr, errstr_len, "Could not measure start time: %s", strerror(errno));
 706                 goto err_open;
 707         }
 708
 709         while (readtransactionfile || txqueue.numitems || inflight.numitems) {
 710                 int ret;
 711
 712                 uint32_t magic;
 713                 uint64_t hand;
 714                 uint32_t command;
 715                 uint64_t from;
 716                 uint32_t len;
 717                 struct reqcontext * prc;
 718
 719                 *errstr=0;
 720
 721                 FD_ZERO(&wset);
 722                 FD_ZERO(&rset);
 723                 if (readtransactionfile)
 724                         FD_SET(logfd, &rset);
 725                 if (txqueue.numitems)
 726                         FD_SET(sock, &wset);
 727                 if (inflight.numitems)
 728                         FD_SET(sock, &rset);
 729                 tv.tv_sec=5;
 730                 tv.tv_usec=0;
 731                 ret = select(1+((sock>logfd)?sock:logfd), &rset, &wset, NULL, &tv);
 732                 if (ret == 0) {
 733                         retval=-1;
 734                         snprintf(errstr, errstr_len, "Timeout reading from socket");
 735                         goto err_open;
 736                 } else if (ret<0) {
 737                         g_warning("Could not mmap temp file: %s", errstr);
 738                         retval=-1;
 739                         goto err;
 740                 }
 741                 /* We know we've got at least one thing to do here then */
 742
 743                 /* Get a command from the transaction log */
 744                 if (FD_ISSET(logfd, &rset)) {
 745
 746                         /* Read a request or reply from the transaction file */
 747                         READ_ALL_ERRCHK(logfd,
 748                                         &magic,
 749                                         sizeof(magic),
 750                                         err_open,
 751                                         "Could not read transaction log: %s",
 752                                         strerror(errno));
 753                         magic = ntohl(magic);
 754                         switch (magic) {
 755                         case NBD_REQUEST_MAGIC:
 756                                 if (NULL == (prc = calloc(1, sizeof(struct reqcontext)))) {
 757                                         retval=-1;
 758                                         snprintf(errstr, errstr_len, "Could not allocate request");
 759                                         goto err_open;
 760                                 }
 761                                 READ_ALL_ERRCHK(logfd,
 762                                                 sizeof(magic)+(char *)&(prc->req),
 763                                                 sizeof(struct nbd_request)-sizeof(magic),
 764                                                 err_open,
 765                                                 "Could not read transaction log: %s",
 766                                                 strerror(errno));
 767                                 prc->req.magic = htonl(NBD_REQUEST_MAGIC);
 768                                 prc->seq=seq++;
 769                                 if ((ntohl(prc->req.type) & NBD_CMD_MASK_COMMAND) == NBD_CMD_DISC) {
 770                                         /* no more to read; don't enqueue as no reply
 771                                          * we will disconnect manually at the end
 772                                          */
 773                                         readtransactionfile = 0;
 774                                         free (prc);
 775                                 } else {
 776                                         dumpcommand("Enqueuing command", prc->req.type);
 777                                         rclist_addtail(&txqueue, prc);
 778                                 }
 779                                 prc = NULL;
 780                                 break;
 781                         case NBD_REPLY_MAGIC:
 782                                 READ_ALL_ERRCHK(logfd,
 783                                                 sizeof(magic)+(char *)(&rep),
 784                                                 sizeof(struct nbd_reply)-sizeof(magic),
 785                                                 err_open,
 786                                                 "Could not read transaction log: %s",
 787                                                 strerror(errno));
 788
 789                                 if (rep.error) {
 790                                         retval=-1;
 791                                         snprintf(errstr, errstr_len, "Transaction log file contained errored transaction");
 792                                         goto err_open;
 793                                 }
 794
 795                                 /* We do not need to consume data on a read reply as there is
 796                                  * none in the log */
 797                                 break;
 798                         default:
 799                                 retval=-1;
 800                                 snprintf(errstr, errstr_len, "Could not measure start time: %08x", magic);
 801                                 goto err_open;
 802                         }
 803                 }
 804
 805                 /* See if we have a write we can do */
 806                 if (FD_ISSET(sock, &wset))
 807                 {
 808                         prc = txqueue.head;
 809                         if (!prc)
 810                                 g_warning("Socket write FD set but we shouldn't have been interested");
 811                         else
 812                         {
 813
 814                                 rclist_unlink(&txqueue, prc);
 815                                 rclist_addtail(&inflight, prc);
 816
 817                                 if (ntohl(prc->req.magic) != NBD_REQUEST_MAGIC) {
 818                                         retval=-1;
 819                                         g_warning("Asked to write a reply without a magic number");
 820                                         goto err_open;
 821                                 }
 822
 823                                 dumpcommand("Sending command", prc->req.type);
 824                                 command = ntohl(prc->req.type);
 825                                 from = ntohll(prc->req.from);
 826                                 len = ntohl(prc->req.len);
 827                                 /* we rewrite the handle as they otherwise may not be unique */
 828                                 *((uint64_t*)(prc->req.handle))=htonll((uint64_t)prc);
 829                                 WRITE_ALL_ERRCHK(sock,
 830                                                  &(prc->req),
 831                                                  sizeof(struct nbd_request),
 832                                                  err_open,
 833                                                  "Could not write command: %s",
 834                                                  strerror(errno));
 835                                 switch (command & NBD_CMD_MASK_COMMAND) {
 836                                 case NBD_CMD_WRITE:
 837                                         while (len > 0) {
 838                                                 uint64_t blknum = from>>9;
 839                                                 char dbuf[512];
 840                                                 if (from>=size) {
 841                                                         snprintf(errstr, errstr_len, "offset %llx beyond size %llx",
 842                                                                  (long long int) from, (long long int)size);
 843                                                         goto err_open;
 844                                                 }
 845                                                 /* work out what we should be writing */
 846                                                 makebuf(dbuf, prc->seq, blknum);
 847                                                 WRITE_ALL_ERRCHK(sock,
 848                                                                  dbuf,
 849                                                                  512,
 850                                                                  err_open,
 851                                                                  "Could not write data: %s",
 852                                                                  strerror(errno));
 853                                                 from += 512;
 854                                                 len -= 512;
 855                                         }
 856
 857                                 case NBD_CMD_DISC:
 858                                 case NBD_CMD_READ:
 859                                 case NBD_CMD_FLUSH:
 860                                         break;
 861                                 default:
 862                                         retval=-1;
 863                                         snprintf(errstr, errstr_len, "Incomprehensible command: %08x", command);
 864                                         goto err_open;
 865                                         break;
 866                                 }
 867
 868                                 prc = NULL;
 869                         }
 870
 871                 }
 872
 873                 /* See if there is a reply to be processed from the socket */
 874                 if(FD_ISSET(sock, &rset)) {
 875                         /* Okay, there's something ready for
 876                          * reading here */
 877
 878                         READ_ALL_ERRCHK(sock,
 879                                         &rep,
 880                                         sizeof(struct nbd_reply),
 881                                         err_open,
 882                                         "Could not read from server socket: %s",
 883                                         strerror(errno));
 884
 885                         if (rep.magic != htonl(NBD_REPLY_MAGIC)) {
 886                                 retval=-1;
 887                                 snprintf(errstr, errstr_len, "Bad magic from server");
 888                                 goto err_open;
 889                         }
 890
 891                         if (rep.error) {
 892                                 retval=-1;
 893                                 snprintf(errstr, errstr_len, "Server errored a transaction");
 894                                 goto err_open;
 895                         }
 896
 897                         prc=(struct reqcontext *)ntohll(*((uint64_t *)rep.handle));
 898                         if (prc->req.magic != htonl(NBD_REQUEST_MAGIC)) {
 899                                 retval=-1;
 900                                 snprintf(errstr, errstr_len, "Bad magic in inflight data: %08x", prc->req.magic);
 901                                 goto err_open;
 902                         }
 903
 904                         dumpcommand("Processing reply to command", prc->req.type);
 905                         command = ntohl(prc->req.type);
 906                         from = ntohll(prc->req.from);
 907                         len = ntohl(prc->req.len);
 908
 909                         switch (command & NBD_CMD_MASK_COMMAND) {
 910                         case NBD_CMD_READ:
 911                                 while (len > 0) {
 912                                         uint64_t blknum = from>>9;
 913                                         char dbuf[512];
 914                                         if (from>=size) {
 915                                                 snprintf(errstr, errstr_len, "offset %llx beyond size %llx",
 916                                                          (long long int) from, (long long int)size);
 917                                                 goto err_open;
 918                                         }
 919                                         READ_ALL_ERRCHK(sock,
 920                                                         dbuf,
 921                                                         512,
 922                                                         err_open,
 923                                                         "Could not read data: %s",
 924                                                         strerror(errno));
 925                                         /* work out what we was written */
 926                                         if (checkbuf(dbuf, blkhash[blknum], blknum))
 927                                         {
 928                                                 retval=-1;
 929                                                 snprintf(errstr, errstr_len, "Bad reply data: seq %08x", blkhash[blknum]);
 930                                                 goto err_open;
 931
 932                                         }
 933                                         from += 512;
 934                                         len -= 512;
 935                                 }
 936                                 break;
 937                         case NBD_CMD_WRITE:
 938                                 /* subsequent reads should get data with this seq*/
 939                                 while (len > 0) {
 940                                         uint64_t blknum = from>>9;
 941                                         blkhash[blknum]=(uint32_t)(prc->seq);
 942                                         from += 512;
 943                                         len -= 512;
 944                                 }
 945                                 break;
 946                         default:
 947                                 break;
 948                         }
 949
 950                         processed++;
 951                         rclist_unlink(&inflight, prc);
 952                         prc->req.magic=0; /* so a duplicate reply is detected */
 953                         free(prc);
 954                 }
 955
 956                 if (!(printer++ % 10000) || !(readtransactionfile || txqueue.numitems || inflight.numitems) )
 957                         printf("%d: Seq %08lld Queued: %08d Inflight: %08d Done: %08lld\n",
 958                                (int)mypid,
 959                                (long long int) seq,
 960                                txqueue.numitems,
 961                                inflight.numitems,
 962                                (long long int) processed);
 963
 964         }
 965
 966         if (gettimeofday(&stop, NULL)<0) {
 967                 retval=-1;
 968                 snprintf(errstr, errstr_len, "Could not measure end time: %s", strerror(errno));
 969                 goto err_open;
 970         }
 971         timespan=timeval_diff_to_double(&stop, &start);
 972         speed=size/timespan;
 973         if(speed>1024) {
 974                 speed=speed/1024.0;
 975                 speedchar[0]='K';
 976         }
 977         if(speed>1024) {
 978                 speed=speed/1024.0;
 979                 speedchar[0]='M';
 980         }
 981         if(speed>1024) {
 982                 speed=speed/1024.0;
 983                 speedchar[0]='G';
 984         }
 985         g_message("%d: Integrity %s test complete. Took %.3f seconds to complete, %.3f%sib/s", (int)getpid(), (testflags & TEST_WRITE)?"write":"read", timespan, speed, speedchar);
 986
 987 err_open:
 988         if(close_sock) {
 989                 close_connection(sock, CONNECTION_CLOSE_PROPERLY);
 990         }
 991 err:
 992         if (size && blkhash)
 993                 munmap(blkhash, (size>>9)<<2);
 994
 995         if (blkhashfd != -1)
 996                 close (blkhashfd);
 997
 998         if (logfd != -1)
 999                 close (logfd);
1000
1001         if (blkhashname)
1002                 free(blkhashname);
1003
1004         if (*errstr)
1005                 g_warning("%s",errstr);
1006
1007         return retval;
1008 }
1009
1010 typedef int (*testfunc)(gchar*, int, char*, int, char, char, int);
1011
1012 int main(int argc, char**argv) {
1013         gchar *hostname;
1014         long int p = 0;
1015         char* name = NULL;
1016         int sock=0;
1017         int c;
1018         bool want_port = TRUE;
1019         int nonopt=0;
1020         int testflags=0;
1021         testfunc test = throughput_test;
1022
1023         if(argc<3) {
1024                 g_message("%d: Not enough arguments", (int)getpid());
1025                 g_message("%d: Usage: %s <hostname> <port>", (int)getpid(), argv[0]);
1026                 g_message("%d: Or: %s <hostname> -N <exportname>", (int)getpid(), argv[0]);
1027                 exit(EXIT_FAILURE);
1028         }
1029         logging();
1030         while((c=getopt(argc, argv, "-N:t:owfi"))>=0) {
1031                 switch(c) {
1032                         case 1:
1033                                 switch(nonopt) {
1034                                         case 0:
1035                                                 hostname=g_strdup(optarg);
1036                                                 nonopt++;
1037                                                 break;
1038                                         case 1:
1039                                                 if(want_port)
1040                                                 p=(strtol(argv[2], NULL, 0));
1041                                                 if(p==LONG_MIN||p==LONG_MAX) {
1042                                                         g_critical("Could not parse port number: %s", strerror(errno));
1043                                                         exit(EXIT_FAILURE);
1044                                                 }
1045                                                 break;
1046                                 }
1047                                 break;
1048                         case 'N':
1049                                 name=g_strdup(optarg);
1050                                 p = 10809;
1051                                 want_port = false;
1052                                 break;
1053                         case 't':
1054                                 transactionlog=g_strdup(optarg);
1055                                 break;
1056                         case 'o':
1057                                 test=oversize_test;
1058                                 break;
1059                         case 'w':
1060                                 testflags|=TEST_WRITE;
1061                                 break;
1062                         case 'f':
1063                                 testflags|=TEST_FLUSH;
1064                                 break;
1065                         case 'i':
1066                                 test=integrity_test;
1067                                 break;
1068                 }
1069         }
1070
1071         if(test(hostname, (int)p, name, sock, FALSE, TRUE, testflags)<0) {
1072                 g_warning("Could not run test: %s", errstr);
1073                 exit(EXIT_FAILURE);
1074         }
1075
1076         return 0;
1077 }