SDL 2.0
SDL_test_harness.h File Reference
#include "begin_code.h"
#include "close_code.h"
+ Include dependency graph for SDL_test_harness.h:
+ This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures

struct  SDLTest_TestCaseReference
 
struct  SDLTest_TestSuiteReference
 

Macros

#define TEST_ENABLED   1
 
#define TEST_DISABLED   0
 
#define TEST_ABORTED   -1
 
#define TEST_STARTED   0
 
#define TEST_COMPLETED   1
 
#define TEST_SKIPPED   2
 
#define TEST_RESULT_PASSED   0
 
#define TEST_RESULT_FAILED   1
 
#define TEST_RESULT_NO_ASSERT   2
 
#define TEST_RESULT_SKIPPED   3
 
#define TEST_RESULT_SETUP_FAILURE   4
 

Typedefs

typedef void(* SDLTest_TestCaseSetUpFp) (void *arg)
 
typedef int(* SDLTest_TestCaseFp) (void *arg)
 
typedef void(* SDLTest_TestCaseTearDownFp) (void *arg)
 

Functions

char * SDLTest_GenerateRunSeed (const int length)
 Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z). More...
 
int SDLTest_RunSuites (SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
 Execute a test suite using the given run seed and execution key. More...
 

Detailed Description

Include file for SDL test framework.

This code is a part of the SDL2_test library, not the main SDL library.

Definition in file SDL_test_harness.h.

Macro Definition Documentation

◆ TEST_ABORTED

#define TEST_ABORTED   -1

Definition at line 51 of file SDL_test_harness.h.

◆ TEST_COMPLETED

#define TEST_COMPLETED   1

Definition at line 53 of file SDL_test_harness.h.

◆ TEST_DISABLED

#define TEST_DISABLED   0

Definition at line 48 of file SDL_test_harness.h.

◆ TEST_ENABLED

#define TEST_ENABLED   1

Definition at line 47 of file SDL_test_harness.h.

◆ TEST_RESULT_FAILED

#define TEST_RESULT_FAILED   1

Definition at line 58 of file SDL_test_harness.h.

◆ TEST_RESULT_NO_ASSERT

#define TEST_RESULT_NO_ASSERT   2

Definition at line 59 of file SDL_test_harness.h.

◆ TEST_RESULT_PASSED

#define TEST_RESULT_PASSED   0

Definition at line 57 of file SDL_test_harness.h.

◆ TEST_RESULT_SETUP_FAILURE

#define TEST_RESULT_SETUP_FAILURE   4

Definition at line 61 of file SDL_test_harness.h.

◆ TEST_RESULT_SKIPPED

#define TEST_RESULT_SKIPPED   3

Definition at line 60 of file SDL_test_harness.h.

◆ TEST_SKIPPED

#define TEST_SKIPPED   2

Definition at line 54 of file SDL_test_harness.h.

◆ TEST_STARTED

#define TEST_STARTED   0

Definition at line 52 of file SDL_test_harness.h.

Typedef Documentation

◆ SDLTest_TestCaseFp

typedef int(* SDLTest_TestCaseFp) (void *arg)

Definition at line 67 of file SDL_test_harness.h.

◆ SDLTest_TestCaseSetUpFp

typedef void(* SDLTest_TestCaseSetUpFp) (void *arg)

Definition at line 64 of file SDL_test_harness.h.

◆ SDLTest_TestCaseTearDownFp

typedef void(* SDLTest_TestCaseTearDownFp) (void *arg)

Definition at line 70 of file SDL_test_harness.h.

Function Documentation

◆ SDLTest_GenerateRunSeed()

char * SDLTest_GenerateRunSeed ( const int  length)

Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z).

Note: The returned string needs to be deallocated by the caller.

Parameters
lengthThe length of the seed string to generate
Returns
The generated seed string

Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z).

Note: The returned string needs to be deallocated by the caller.

Parameters
lengthThe length of the seed string to generate
Returns
The generated seed string

Definition at line 54 of file SDL_test_harness.c.

55{
56 char *seed = NULL;
57 SDLTest_RandomContext randomContext;
58 int counter;
59
60 /* Sanity check input */
61 if (length <= 0) {
62 SDLTest_LogError("The length of the harness seed must be >0.");
63 return NULL;
64 }
65
66 /* Allocate output buffer */
67 seed = (char *)SDL_malloc((length + 1) * sizeof(char));
68 if (seed == NULL) {
69 SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
71 return NULL;
72 }
73
74 /* Generate a random string of alphanumeric characters */
75 SDLTest_RandomInitTime(&randomContext);
76 for (counter = 0; counter < length; counter++) {
77 unsigned int number = SDLTest_Random(&randomContext);
78 char ch = (char) (number % (91 - 48)) + 48;
79 if (ch >= 58 && ch <= 64) {
80 ch = 65;
81 }
82 seed[counter] = ch;
83 }
84 seed[length] = '\0';
85
86 return seed;
87}
#define SDL_Error
#define SDL_malloc
@ SDL_ENOMEM
Definition: SDL_error.h:57
GLuint GLsizei GLsizei * length
GLuint counter
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
void SDLTest_RandomInitTime(SDLTest_RandomContext *rndContext)
Initialize random number generator based on current system time.
unsigned int SDLTest_Random(SDLTest_RandomContext *rndContext)
Initialize random number generator based on current system time.
#define NULL
Definition: begin_code.h:167

References NULL, SDL_ENOMEM, SDL_Error, SDL_malloc, SDLTest_LogError(), SDLTest_Random(), and SDLTest_RandomInitTime().

◆ SDLTest_RunSuites()

int SDLTest_RunSuites ( SDLTest_TestSuiteReference testSuites[],
const char *  userRunSeed,
Uint64  userExecKey,
const char *  filter,
int  testIterations 
)

Execute a test suite using the given run seed and execution key.

Parameters
testSuitesSuites containing the test case.
userRunSeedCustom run seed provided by user, or NULL to autogenerate one.
userExecKeyCustom execution key provided by user, or 0 to autogenerate one.
filterFilter specification. NULL disables. Case sensitive.
testIterationsNumber of iterations to run each test case.
Returns
Test run result; 0 when all tests passed, 1 if any tests failed.

The filter string is matched to the suite name (full comparison) to select a single suite, or if no suite matches, it is matched to the test names (full comparison) to select a single test.

Parameters
testSuitesSuites containing the test case.
userRunSeedCustom run seed provided by user, or NULL to autogenerate one.
userExecKeyCustom execution key provided by user, or 0 to autogenerate one.
filterFilter specification. NULL disables. Case sensitive.
testIterationsNumber of iterations to run each test case.
Returns
Test run result; 0 when all tests passed, 1 if any tests failed.

Definition at line 370 of file SDL_test_harness.c.

371{
372 int totalNumberOfTests = 0;
373 int failedNumberOfTests = 0;
374 int suiteCounter;
375 int testCounter;
376 int iterationCounter;
378 const SDLTest_TestCaseReference *testCase;
379 const char *runSeed = NULL;
380 char *currentSuiteName;
381 char *currentTestName;
382 Uint64 execKey;
383 float runStartSeconds;
384 float suiteStartSeconds;
385 float testStartSeconds;
386 float runEndSeconds;
387 float suiteEndSeconds;
388 float testEndSeconds;
389 float runtime;
390 int suiteFilter = 0;
391 char *suiteFilterName = NULL;
392 int testFilter = 0;
393 char *testFilterName = NULL;
394 SDL_bool forceTestRun = SDL_FALSE;
395 int testResult = 0;
396 int runResult = 0;
397 Uint32 totalTestFailedCount = 0;
398 Uint32 totalTestPassedCount = 0;
399 Uint32 totalTestSkippedCount = 0;
400 Uint32 testFailedCount = 0;
401 Uint32 testPassedCount = 0;
402 Uint32 testSkippedCount = 0;
403 Uint32 countSum = 0;
404 const SDLTest_TestCaseReference **failedTests;
405
406 /* Sanitize test iterations */
407 if (testIterations < 1) {
408 testIterations = 1;
409 }
410
411 /* Generate run see if we don't have one already */
412 if (userRunSeed == NULL || userRunSeed[0] == '\0') {
413 runSeed = SDLTest_GenerateRunSeed(16);
414 if (runSeed == NULL) {
415 SDLTest_LogError("Generating a random seed failed");
416 return 2;
417 }
418 } else {
419 runSeed = userRunSeed;
420 }
421
422
423 /* Reset per-run counters */
424 totalTestFailedCount = 0;
425 totalTestPassedCount = 0;
426 totalTestSkippedCount = 0;
427
428 /* Take time - run start */
429 runStartSeconds = GetClock();
430
431 /* Log run with fuzzer parameters */
432 SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
433
434 /* Count the total number of tests */
435 suiteCounter = 0;
436 while (testSuites[suiteCounter]) {
437 testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
438 suiteCounter++;
439 testCounter = 0;
440 while (testSuite->testCases[testCounter])
441 {
442 testCounter++;
443 totalNumberOfTests++;
444 }
445 }
446
447 /* Pre-allocate an array for tracking failed tests (potentially all test cases) */
448 failedTests = (const SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
449 if (failedTests == NULL) {
450 SDLTest_LogError("Unable to allocate cache for failed tests");
452 return -1;
453 }
454
455 /* Initialize filtering */
456 if (filter != NULL && filter[0] != '\0') {
457 /* Loop over all suites to check if we have a filter match */
458 suiteCounter = 0;
459 while (testSuites[suiteCounter] && suiteFilter == 0) {
460 testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
461 suiteCounter++;
462 if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
463 /* Matched a suite name */
464 suiteFilter = 1;
465 suiteFilterName = testSuite->name;
466 SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
467 break;
468 }
469
470 /* Within each suite, loop over all test cases to check if we have a filter match */
471 testCounter = 0;
472 while (testSuite->testCases[testCounter] && testFilter == 0)
473 {
474 testCase = testSuite->testCases[testCounter];
475 testCounter++;
476 if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
477 /* Matched a test name */
478 suiteFilter = 1;
479 suiteFilterName = testSuite->name;
480 testFilter = 1;
481 testFilterName = testCase->name;
482 SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
483 break;
484 }
485 }
486 }
487
488 if (suiteFilter == 0 && testFilter == 0) {
489 SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
490 SDLTest_Log("Exit code: 2");
491 SDL_free((void *) failedTests);
492 return 2;
493 }
494 }
495
496 /* Loop over all suites */
497 suiteCounter = 0;
498 while(testSuites[suiteCounter]) {
499 testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
500 currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
501 suiteCounter++;
502
503 /* Filter suite if flag set and we have a name */
504 if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
505 SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
506 /* Skip suite */
507 SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
508 suiteCounter,
509 currentSuiteName);
510 } else {
511
512 /* Reset per-suite counters */
513 testFailedCount = 0;
514 testPassedCount = 0;
515 testSkippedCount = 0;
516
517 /* Take time - suite start */
518 suiteStartSeconds = GetClock();
519
520 /* Log suite started */
521 SDLTest_Log("===== Test Suite %i: '%s' started\n",
522 suiteCounter,
523 currentSuiteName);
524
525 /* Loop over all test cases */
526 testCounter = 0;
527 while(testSuite->testCases[testCounter])
528 {
529 testCase = testSuite->testCases[testCounter];
530 currentTestName = (char *)((testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT);
531 testCounter++;
532
533 /* Filter tests if flag set and we have a name */
534 if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
535 SDL_strcmp(testFilterName, testCase->name) != 0) {
536 /* Skip test */
537 SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
538 suiteCounter,
539 testCounter,
540 currentTestName);
541 } else {
542 /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
543 if (testFilter == 1 && !testCase->enabled) {
544 SDLTest_Log("Force run of disabled test since test filter was set");
545 forceTestRun = SDL_TRUE;
546 }
547
548 /* Take time - test start */
549 testStartSeconds = GetClock();
550
551 /* Log test started */
552 SDLTest_Log("----- Test Case %i.%i: '%s' started",
553 suiteCounter,
554 testCounter,
555 currentTestName);
556 if (testCase->description != NULL && testCase->description[0] != '\0') {
557 SDLTest_Log("Test Description: '%s'",
558 (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
559 }
560
561 /* Loop over all iterations */
562 iterationCounter = 0;
563 while(iterationCounter < testIterations)
564 {
565 iterationCounter++;
566
567 if (userExecKey != 0) {
568 execKey = userExecKey;
569 } else {
570 execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
571 }
572
573 SDLTest_Log("Test Iteration %i: execKey %" SDL_PRIu64, iterationCounter, execKey);
574 testResult = SDLTest_RunTest(testSuite, testCase, execKey, forceTestRun);
575
576 if (testResult == TEST_RESULT_PASSED) {
577 testPassedCount++;
578 totalTestPassedCount++;
579 } else if (testResult == TEST_RESULT_SKIPPED) {
580 testSkippedCount++;
581 totalTestSkippedCount++;
582 } else {
583 testFailedCount++;
584 totalTestFailedCount++;
585 }
586 }
587
588 /* Take time - test end */
589 testEndSeconds = GetClock();
590 runtime = testEndSeconds - testStartSeconds;
591 if (runtime < 0.0f) runtime = 0.0f;
592
593 if (testIterations > 1) {
594 /* Log test runtime */
595 SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
596 SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
597 } else {
598 /* Log test runtime */
599 SDLTest_Log("Total Test runtime: %.1f sec", runtime);
600 }
601
602 /* Log final test result */
603 switch (testResult) {
605 SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Passed");
606 break;
608 SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Failed");
609 break;
611 SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT,"Test", currentTestName, "No Asserts");
612 break;
613 }
614
615 /* Collect failed test case references for repro-step display */
616 if (testResult == TEST_RESULT_FAILED) {
617 failedTests[failedNumberOfTests] = testCase;
618 failedNumberOfTests++;
619 }
620 }
621 }
622
623 /* Take time - suite end */
624 suiteEndSeconds = GetClock();
625 runtime = suiteEndSeconds - suiteStartSeconds;
626 if (runtime < 0.0f) runtime = 0.0f;
627
628 /* Log suite runtime */
629 SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
630
631 /* Log summary and final Suite result */
632 countSum = testPassedCount + testFailedCount + testSkippedCount;
633 if (testFailedCount == 0)
634 {
635 SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
636 SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Passed");
637 }
638 else
639 {
640 SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
641 SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Failed");
642 }
643
644 }
645 }
646
647 /* Take time - run end */
648 runEndSeconds = GetClock();
649 runtime = runEndSeconds - runStartSeconds;
650 if (runtime < 0.0f) runtime = 0.0f;
651
652 /* Log total runtime */
653 SDLTest_Log("Total Run runtime: %.1f sec", runtime);
654
655 /* Log summary and final run result */
656 countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
657 if (totalTestFailedCount == 0)
658 {
659 runResult = 0;
660 SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
661 SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Passed");
662 }
663 else
664 {
665 runResult = 1;
666 SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
667 SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Failed");
668 }
669
670 /* Print repro steps for failed tests */
671 if (failedNumberOfTests > 0) {
672 SDLTest_Log("Harness input to repro failures:");
673 for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
674 SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
675 }
676 }
677 SDL_free((void *) failedTests);
678
679 SDLTest_Log("Exit code: %d", runResult);
680 return runResult;
681}
#define SDL_free
#define SDL_strcmp
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter
GLuint const GLchar * name
SDL_bool
Definition: SDL_stdinc.h:162
@ SDL_TRUE
Definition: SDL_stdinc.h:164
@ SDL_FALSE
Definition: SDL_stdinc.h:163
#define SDL_PRIu64
Definition: SDL_stdinc.h:238
uint32_t Uint32
Definition: SDL_stdinc.h:203
uint64_t Uint64
Definition: SDL_stdinc.h:216
char * SDLTest_GenerateRunSeed(const int length)
Generates a random run seed string for the harness. The generated seed will contain alphanumeric char...
static Uint64 SDLTest_GenerateExecKey(const char *runSeed, char *suiteName, char *testName, int iteration)
#define SDLTEST_INVALID_NAME_FORMAT
#define SDLTEST_FINAL_RESULT_FORMAT
#define SDLTEST_LOG_SUMMARY_FORMAT
static int SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, const SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
Execute a test using the given execution key.
static float GetClock()
#define TEST_RESULT_FAILED
#define TEST_RESULT_SKIPPED
#define TEST_RESULT_NO_ASSERT
#define TEST_RESULT_PASSED
void SDLTest_Log(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and INFO priority.
Definition: SDL_test_log.c:85
const SDLTest_TestCaseReference ** testCases
SDLTest_TestSuiteReference * testSuites[]

References SDLTest_TestCaseReference::description, SDLTest_TestCaseReference::enabled, GetClock(), SDLTest_TestCaseReference::name, SDLTest_TestSuiteReference::name, NULL, SDL_ENOMEM, SDL_Error, SDL_FALSE, SDL_free, SDL_malloc, SDL_PRIu64, SDL_strcmp, SDL_TRUE, SDLTEST_FINAL_RESULT_FORMAT, SDLTest_GenerateExecKey(), SDLTest_GenerateRunSeed(), SDLTEST_INVALID_NAME_FORMAT, SDLTest_Log(), SDLTEST_LOG_SUMMARY_FORMAT, SDLTest_LogError(), SDLTest_RunTest(), TEST_RESULT_FAILED, TEST_RESULT_NO_ASSERT, TEST_RESULT_PASSED, TEST_RESULT_SKIPPED, SDLTest_TestSuiteReference::testCases, and testSuites.

Referenced by main().