ECE 408 sem project
This commit is contained in:
Aryan Singh committed
1 parent
1d79ccd2e3
commit
fa3efcaf07
139 files changed
+11634
-1
No files matched your search
@@ -0,0 +1,48 @@
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
// Copyright (c) 2024 Andrej Karpathy
|
||||
// Licensed under the MIT License. See the LICENSE file for details.
|
||||
//
|
||||
// Modifications Copyright (c) 2025 Hanwen Liu, Hrishi Shah, Kelin Zeng, Charles Pei, and Vijay Daita, ALL RIGHTS RESERVED.
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
|
||||
/*
|
||||
Helpful CUDA utilities for error checking and other common operations.
|
||||
*/
|
||||
#ifndef CUDA_UTILS_CUH
|
||||
#define CUDA_UTILS_CUH
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <float.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <cublas_v2.h>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
// convenience macro for calculating grid/block dimensions for kernels
|
||||
#define CEIL_DIV(M, N) (((M) + (N)-1) / (N))
|
||||
|
||||
// CUDA error checking
|
||||
void cudaCheck(cudaError_t error, const char *file, int line) {
|
||||
if (error != cudaSuccess) {
|
||||
printf("[CUDA ERROR] at file %s:%d:\n%s\n", file, line,
|
||||
cudaGetErrorString(error));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
};
|
||||
#define cudaCheck(err) (cudaCheck(err, __FILE__, __LINE__))
|
||||
|
||||
// cuBLAS error checking
|
||||
void cublasCheck(cublasStatus_t status, const char *file, int line)
|
||||
{
|
||||
if (status != CUBLAS_STATUS_SUCCESS) {
|
||||
printf("[cuBLAS ERROR]: %d %s %d\n", status, file, line);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
#define cublasCheck(status) { cublasCheck((status), __FILE__, __LINE__); }
|
||||
|
||||
static cublasComputeType_t cublas_compute_type;
|
||||
cublasHandle_t cublas_handle;
|
||||
|
||||
#endif // CUDA_UTILS_CUH
|
||||
@@ -0,0 +1,108 @@
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
// Copyright (c) 2024 Andrej Karpathy
|
||||
// Licensed under the MIT License. See the LICENSE file for details.
|
||||
//
|
||||
// Modifications Copyright (c) 2025 Hanwen Liu, Hrishi Shah, Kelin Zeng, Charles Pei, and Vijay Daita, ALL RIGHTS RESERVED.
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
|
||||
/*
|
||||
Defines the GPT-2 Tokenizer.
|
||||
Only supports decoding, i.e.: tokens (integers) -> strings
|
||||
*/
|
||||
#ifndef UTILS_TOKENIZER_H
|
||||
#define UTILS_TOKENIZER_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <ctype.h>
|
||||
#include <assert.h>
|
||||
#include "utils.h"
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
typedef struct {
|
||||
uint32_t vocab_size;
|
||||
char **token_table;
|
||||
int init_ok;
|
||||
int eot_token; // <|endoftext|> token id
|
||||
} Tokenizer;
|
||||
|
||||
void safe_printf(const char *piece) {
|
||||
// the tokens are raw bytes, and we we only want to print the printable ones
|
||||
// many bytes can be various control codes, backspace, etc.
|
||||
if (piece == NULL) { return; }
|
||||
if (piece[0] == '\0') { return; }
|
||||
// handle individual byte tokens
|
||||
// every token is asserted to be at least one byte so doing piece[1] is ok
|
||||
if (piece[1] == '\0') {
|
||||
unsigned char byte_val = piece[0];
|
||||
if (!(isprint(byte_val) || isspace(byte_val))) {
|
||||
return; // weird byte, don't print it
|
||||
}
|
||||
}
|
||||
printf("%s", piece);
|
||||
}
|
||||
|
||||
void tokenizer_init(Tokenizer *tokenizer, const char *filename) {
|
||||
FILE *file = fopen(filename, "rb");
|
||||
if (file == NULL) {
|
||||
printf("---\n");
|
||||
printf("Tokenizer - WARNING: Failed to open the tokenizer file %s\n", filename);
|
||||
printf("---\n");
|
||||
tokenizer->init_ok = 0;
|
||||
return;
|
||||
}
|
||||
// read in the header
|
||||
uint32_t header[256];
|
||||
freadCheck(header, sizeof(uint32_t), 256, file);
|
||||
assert(header[0] == 20240328);
|
||||
int version = header[1];
|
||||
tokenizer->vocab_size = header[2];
|
||||
if (version == 1) {
|
||||
// version 1 didn't include the EOT token id
|
||||
// so we assume it is 50256, the EOT in GPT-2
|
||||
assert(tokenizer->vocab_size == 50257); // let's be defensive here
|
||||
tokenizer->eot_token = 50256;
|
||||
} else if (version == 2) {
|
||||
tokenizer->eot_token = header[3];
|
||||
} else {
|
||||
fprintf(stderr, "Tokenizer model file %s has bad version: %d\n", filename, version);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
// read in all the tokens
|
||||
unsigned char length;
|
||||
tokenizer->token_table = (char **)mallocCheck(tokenizer->vocab_size * sizeof(char *));
|
||||
for (uint32_t i = 0; i < tokenizer->vocab_size; i++) {
|
||||
freadCheck(&length, sizeof(unsigned char), 1, file);
|
||||
assert(length > 0); // every token should be at least one character
|
||||
char *token_bytes = (char *)mallocCheck(length + 1);
|
||||
freadCheck(token_bytes, sizeof(char), length, file);
|
||||
token_bytes[length] = '\0'; // Add null terminator for printing
|
||||
tokenizer->token_table[i] = token_bytes;
|
||||
}
|
||||
// cleanups
|
||||
fcloseCheck(file);
|
||||
tokenizer->init_ok = 1;
|
||||
}
|
||||
|
||||
const char *tokenizer_decode(Tokenizer *tokenizer, uint32_t token_id) {
|
||||
if (tokenizer->init_ok == 0) {
|
||||
return NULL;
|
||||
}
|
||||
if (token_id < tokenizer->vocab_size) {
|
||||
return tokenizer->token_table[token_id];
|
||||
} else {
|
||||
printf("Tokenizer - Invalid token id %u!\n", token_id);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
void tokenizer_free(Tokenizer *tokenizer) {
|
||||
if (tokenizer->init_ok) {
|
||||
for (uint32_t i = 0; i < tokenizer->vocab_size; i++) {
|
||||
free(tokenizer->token_table[i]);
|
||||
}
|
||||
free(tokenizer->token_table);
|
||||
}
|
||||
}
|
||||
|
||||
#endif // UTILS_TOKENIZER_H
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
// Copyright (c) 2024 Andrej Karpathy
|
||||
// Licensed under the MIT License. See the LICENSE file for details.
|
||||
//
|
||||
// Modifications Copyright (c) 2025 Hanwen Liu, Hrishi Shah, Kelin Zeng, Charles Pei, and Vijay Daita, ALL RIGHTS RESERVED.
|
||||
//-----------------------------------------------------------------------------------------------
|
||||
|
||||
#ifndef UTILS_H
|
||||
#define UTILS_H
|
||||
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/stat.h>
|
||||
#ifndef _WIN32
|
||||
#include <dirent.h>
|
||||
#endif
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// fread convenience utils, with nice handling of error checking using macros
|
||||
// simple replace fopen, fread, fclose, fseek
|
||||
// with fopenCheck, freadCheck, fcloseCheck, fseekCheck
|
||||
|
||||
FILE *fopen_check(const char *path, const char *mode, const char *file, int line) {
|
||||
FILE *fp = fopen(path, mode);
|
||||
if (fp == NULL) {
|
||||
fprintf(stderr, "Utils - Error: Failed to open file '%s' at %s:%d\n", path, file, line);
|
||||
fprintf(stderr, "Error details:\n");
|
||||
fprintf(stderr, " File: %s\n", file);
|
||||
fprintf(stderr, " Line: %d\n", line);
|
||||
fprintf(stderr, " Path: %s\n", path);
|
||||
fprintf(stderr, " Mode: %s\n", mode);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
return fp;
|
||||
}
|
||||
|
||||
#define fopenCheck(path, mode) fopen_check(path, mode, __FILE__, __LINE__)
|
||||
|
||||
void fread_check(void *ptr, size_t size, size_t nmemb, FILE *stream, const char *file, int line) {
|
||||
size_t result = fread(ptr, size, nmemb, stream);
|
||||
if (result != nmemb) {
|
||||
if (feof(stream)) {
|
||||
fprintf(stderr, "Utils - Error: Unexpected end of file at %s:%d\n", file, line);
|
||||
} else if (ferror(stream)) {
|
||||
fprintf(stderr, "Utils - Error: File read error at %s:%d\n", file, line);
|
||||
} else {
|
||||
fprintf(stderr, "Utils - Error: Partial read at %s:%d. Expected %zu elements, read %zu\n",
|
||||
file, line, nmemb, result);
|
||||
}
|
||||
fprintf(stderr, "Utils - Error details:\n");
|
||||
fprintf(stderr, " File: %s\n", file);
|
||||
fprintf(stderr, " Line: %d\n", line);
|
||||
fprintf(stderr, " Expected elements: %zu\n", nmemb);
|
||||
fprintf(stderr, " Read elements: %zu\n", result);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
#define freadCheck(ptr, size, nmemb, stream) fread_check(ptr, size, nmemb, stream, __FILE__, __LINE__)
|
||||
|
||||
void fclose_check(FILE *fp, const char *file, int line) {
|
||||
if (fclose(fp) != 0) {
|
||||
fprintf(stderr, "Utils - Error: Failed to close file at %s:%d\n", file, line);
|
||||
fprintf(stderr, "Error details:\n");
|
||||
fprintf(stderr, " File: %s\n", file);
|
||||
fprintf(stderr, " Line: %d\n", line);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
#define fcloseCheck(fp) fclose_check(fp, __FILE__, __LINE__)
|
||||
|
||||
void fseek_check(FILE *fp, long off, int whence, const char *file, int line) {
|
||||
if (fseek(fp, off, whence) != 0) {
|
||||
fprintf(stderr, "Utils - Error: Failed to seek in file at %s:%d\n", file, line);
|
||||
fprintf(stderr, "Error details:\n");
|
||||
fprintf(stderr, " Offset: %ld\n", off);
|
||||
fprintf(stderr, " Whence: %d\n", whence);
|
||||
fprintf(stderr, " File: %s\n", file);
|
||||
fprintf(stderr, " Line: %d\n", line);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
#define fseekCheck(fp, off, whence) fseek_check(fp, off, whence, __FILE__, __LINE__)
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// malloc error-handling wrapper util
|
||||
|
||||
void *malloc_check(size_t size, const char *file, int line) {
|
||||
void *ptr = malloc(size);
|
||||
if (ptr == NULL) {
|
||||
fprintf(stderr, "Utils - Error: Memory allocation failed at %s:%d\n", file, line);
|
||||
fprintf(stderr, "Error details:\n");
|
||||
fprintf(stderr, " File: %s\n", file);
|
||||
fprintf(stderr, " Line: %d\n", line);
|
||||
fprintf(stderr, " Size: %zu bytes\n", size);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
return ptr;
|
||||
}
|
||||
|
||||
#define mallocCheck(size) malloc_check(size, __FILE__, __LINE__)
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// I/O ops
|
||||
|
||||
void create_dir_if_not_exists(const char *dir) {
|
||||
if (dir == NULL) { return; }
|
||||
struct stat st = {0};
|
||||
if (stat(dir, &st) == -1) {
|
||||
if (mkdir(dir, 0700) == -1) {
|
||||
printf("Utils - ERROR: could not create directory: %s\n", dir);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
printf("Utils - Created directory: %s\n", dir);
|
||||
}
|
||||
}
|
||||
|
||||
int find_max_step(const char* output_log_dir) {
|
||||
// find the DONE file in the log dir with highest step count
|
||||
if (output_log_dir == NULL) { return -1; }
|
||||
DIR* dir;
|
||||
struct dirent* entry;
|
||||
int max_step = -1;
|
||||
dir = opendir(output_log_dir);
|
||||
if (dir == NULL) { return -1; }
|
||||
while ((entry = readdir(dir)) != NULL) {
|
||||
if (strncmp(entry->d_name, "DONE_", 5) == 0) {
|
||||
int step = atoi(entry->d_name + 5);
|
||||
if (step > max_step) {
|
||||
max_step = step;
|
||||
}
|
||||
}
|
||||
}
|
||||
closedir(dir);
|
||||
return max_step;
|
||||
}
|
||||
|
||||
#endif
|
||||
Reference in new issue
Block a user