CUPTI PC Sampling Start/Stop Control Tutorial

cd pc_sampling_start_stop
make

./pc_sampling_start_stop

class PCSamplingController {
private:
    struct SamplingSession {
        std::string phaseName;
        uint64_t startTime;
        uint64_t endTime;
        uint32_t samplesCollected;
        bool isActive;
    };
    
    std::vector<SamplingSession> sessions;
    std::atomic<bool> samplingActive;
    std::mutex controlMutex;
    CUpti_PCSamplingConfig config;
 
public:
    void initializeSampling();
    void startSampling(const std::string& phaseName);
    void stopSampling();
    void conditionalStart(std::function<bool()> condition);
    void conditionalStop(std::function<bool()> condition);
    void generatePhaseReport();
};
 
void PCSamplingController::startSampling(const std::string& phaseName) {
    std::lock_guard<std::mutex> lock(controlMutex);
    
    if (samplingActive.load()) {
        std::cout << "Warning: Sampling already active, stopping previous session" << std::endl;
        stopSamplingInternal();
    }
    
    SamplingSession session;
    session.phaseName = phaseName;
    session.startTime = getCurrentTimestamp();
    session.isActive = true;
    
    // Start PC sampling
    CUPTI_CALL(cuptiPCSamplingStart(context, &config));
    
    samplingActive = true;
    sessions.push_back(session);
    
    std::cout << "PC Sampling started for phase: " << phaseName 
             << " at t=" << session.startTime << "ms" << std::endl;
}
 
void PCSamplingController::stopSampling() {
    std::lock_guard<std::mutex> lock(controlMutex);
    stopSamplingInternal();
}
 
void PCSamplingController::stopSamplingInternal() {
    if (!samplingActive.load()) {
        return;
    }
    
    // Stop PC sampling
    CUPTI_CALL(cuptiPCSamplingStop(context));
    
    // Update the last session
    if (!sessions.empty()) {
        auto& lastSession = sessions.back();
        lastSession.endTime = getCurrentTimestamp();
        lastSession.isActive = false;
        
        // Collect samples from this session
        lastSession.samplesCollected = collectPCSamples();
        
        std::cout << "PC Sampling stopped for phase: " << lastSession.phaseName
                 << " at t=" << lastSession.endTime << "ms" << std::endl;
        std::cout << "  Samples collected: " << lastSession.samplesCollected << std::endl;
    }
    
    samplingActive = false;
}

class ConditionalSamplingManager {
private:
    struct SamplingCondition {
        std::string name;
        std::function<bool()> startCondition;
        std::function<bool()> stopCondition;
        bool isMonitoring;
        std::thread monitorThread;
    };
    
    std::vector<SamplingCondition> conditions;
    PCSamplingController& controller;
    std::atomic<bool> monitoringActive;
 
public:
    ConditionalSamplingManager(PCSamplingController& ctrl) : controller(ctrl), monitoringActive(false) {}
    
    void addCondition(const std::string& name,
                     std::function<bool()> startCond,
                     std::function<bool()> stopCond) {
        SamplingCondition condition;
        condition.name = name;
        condition.startCondition = startCond;
        condition.stopCondition = stopCond;
        condition.isMonitoring = false;
        
        conditions.push_back(condition);
    }
    
    void startMonitoring() {
        monitoringActive = true;
        
        for (auto& condition : conditions) {
            condition.isMonitoring = true;
            condition.monitorThread = std::thread([this, &condition]() {
                monitorCondition(condition);
            });
        }
    }
    
    void stopMonitoring() {
        monitoringActive = false;
        
        for (auto& condition : conditions) {
            condition.isMonitoring = false;
            if (condition.monitorThread.joinable()) {
                condition.monitorThread.join();
            }
        }
    }
    
private:
    void monitorCondition(SamplingCondition& condition) {
        bool samplingForThisCondition = false;
        
        while (condition.isMonitoring && monitoringActive) {
            if (!samplingForThisCondition && condition.startCondition()) {
                controller.startSampling(condition.name);
                samplingForThisCondition = true;
                std::cout << "Conditional sampling started: " << condition.name << std::endl;
            } else if (samplingForThisCondition && condition.stopCondition()) {
                controller.stopSampling();
                samplingForThisCondition = false;
                std::cout << "Conditional sampling stopped: " << condition.name << std::endl;
            }
            
            std::this_thread::sleep_for(std::milliseconds(10));
        }
    }
};

class PerformanceTriggeredSampling {
private:
    struct PerformanceMetrics {
        double gpuUtilization;
        double memoryBandwidth;
        double computeEfficiency;
        uint64_t timestamp;
    };
    
    std::queue<PerformanceMetrics> metricsHistory;
    PCSamplingController& controller;
    double utilizationThreshold;
    double bandwidthThreshold;
    bool autoSamplingEnabled;
 
public:
    PerformanceTriggeredSampling(PCSamplingController& ctrl) 
        : controller(ctrl), utilizationThreshold(0.7), bandwidthThreshold(0.5), autoSamplingEnabled(false) {}
    
    void enableAutoSampling(double utilThreshold, double bwThreshold) {
        utilizationThreshold = utilThreshold;
        bandwidthThreshold = bwThreshold;
        autoSamplingEnabled = true;
    }
    
    void updateMetrics(double utilization, double bandwidth, double efficiency) {
        PerformanceMetrics metrics;
        metrics.gpuUtilization = utilization;
        metrics.memoryBandwidth = bandwidth;
        metrics.computeEfficiency = efficiency;
        metrics.timestamp = getCurrentTimestamp();
        
        metricsHistory.push(metrics);
        
        // Keep only recent history
        while (metricsHistory.size() > 100) {
            metricsHistory.pop();
        }
        
        if (autoSamplingEnabled) {
            evaluateSamplingTriggers(metrics);
        }
    }
    
private:
    void evaluateSamplingTriggers(const PerformanceMetrics& current) {
        // Trigger sampling during high-utilization periods
        if (current.gpuUtilization > utilizationThreshold && 
            current.memoryBandwidth > bandwidthThreshold) {
            
            if (!controller.isSamplingActive()) {
                controller.startSampling("High Performance Period");
            }
        }
        // Stop sampling during low-utilization periods
        else if (current.gpuUtilization < utilizationThreshold * 0.7) {
            if (controller.isSamplingActive()) {
                controller.stopSampling();
            }
        }
        
        // Trigger sampling when efficiency drops (potential issue)
        if (current.computeEfficiency < 0.6 && !controller.isSamplingActive()) {
            controller.startSampling("Low Efficiency Investigation");
        }
    }
};

class KernelSamplingController {
private:
    std::set<std::string> targetKernels;
    std::map<std::string, uint32_t> kernelSampleCounts;
    PCSamplingController& controller;
 
public:
    KernelSamplingController(PCSamplingController& ctrl) : controller(ctrl) {}
    
    void addTargetKernel(const std::string& kernelName) {
        targetKernels.insert(kernelName);
    }
    
    void onKernelLaunch(const std::string& kernelName) {
        if (targetKernels.find(kernelName) != targetKernels.end()) {
            controller.startSampling("Kernel: " + kernelName);
        }
    }
    
    void onKernelComplete(const std::string& kernelName) {
        if (targetKernels.find(kernelName) != targetKernels.end()) {
            controller.stopSampling();
            kernelSampleCounts[kernelName]++;
        }
    }
    
    void generateKernelReport() {
        std::cout << "Kernel Sampling Summary:" << std::endl;
        for (const auto& [kernelName, count] : kernelSampleCounts) {
            std::cout << "  " << kernelName << ": sampled " << count << " times" << std::endl;
        }
    }
};

class TimeWindowSampling {
private:
    struct SamplingWindow {
        uint64_t startTime;
        uint64_t duration;
        uint64_t interval;
        bool isActive;
        std::thread windowThread;
    };
    
    std::vector<SamplingWindow> windows;
    PCSamplingController& controller;
    std::atomic<bool> windowingActive;
 
public:
    TimeWindowSampling(PCSamplingController& ctrl) : controller(ctrl), windowingActive(false) {}
    
    void addPeriodicWindow(uint64_t durationMs, uint64_t intervalMs) {
        SamplingWindow window;
        window.duration = durationMs;
        window.interval = intervalMs;
        window.isActive = false;
        
        windows.push_back(window);
    }
    
    void startWindowedSampling() {
        windowingActive = true;
        
        for (auto& window : windows) {
            window.isActive = true;
            window.windowThread = std::thread([this, &window]() {
                executeWindowedSampling(window);
            });
        }
    }
    
    void stopWindowedSampling() {
        windowingActive = false;
        
        for (auto& window : windows) {
            window.isActive = false;
            if (window.windowThread.joinable()) {
                window.windowThread.join();
            }
        }
    }
    
private:
    void executeWindowedSampling(SamplingWindow& window) {
        while (window.isActive && windowingActive) {
            // Start sampling for this window
            controller.startSampling("Time Window");
            
            // Sample for the specified duration
            std::this_thread::sleep_for(std::milliseconds(window.duration));
            
            // Stop sampling
            controller.stopSampling();
            
            // Wait for the interval period
            std::this_thread::sleep_for(std::milliseconds(window.interval - window.duration));
        }
    }
};

void profileIterativeAlgorithm() {
    PCSamplingController controller;
    ConditionalSamplingManager condManager(controller);
    
    // Set up conditions for sampling
    condManager.addCondition("Convergence Issues",
        []() { return getCurrentError() > ERROR_THRESHOLD; },  // Start condition
        []() { return getCurrentError() < ERROR_THRESHOLD; }   // Stop condition
    );
    
    condManager.addCondition("High Iteration Count",
        []() { return getCurrentIteration() > 1000; },         // Start condition
        []() { return getCurrentIteration() > 1500; }          // Stop condition (timeout)
    );
    
    condManager.startMonitoring();
    
    // Run iterative algorithm
    for (int i = 0; i < maxIterations; i++) {
        performIteration();
        
        // Manual control for specific phases
        if (i % 100 == 0) {
            controller.startSampling("Checkpoint Iteration " + std::to_string(i));
            performDetailedAnalysis();
            controller.stopSampling();
        }
    }
    
    condManager.stopMonitoring();
    controller.generatePhaseReport();
}

void analyzeMultiPhaseApplication() {
    PCSamplingController controller;
    PerformanceTriggeredSampling perfSampling(controller);
    
    // Enable performance-triggered sampling
    perfSampling.enableAutoSampling(0.8, 0.6); // 80% GPU, 60% bandwidth thresholds
    
    // Phase 1: Data preprocessing
    // (No manual sampling - let performance triggers handle it)
    preprocessData();
    
    // Phase 2: Core computation (always sample)
    controller.startSampling("Core Computation");
    performCoreComputation();
    controller.stopSampling();
    
    // Phase 3: Optimization loop (conditional sampling)
    for (int iter = 0; iter < optimizationIterations; iter++) {
        if (iter % 10 == 0 || isConvergenceSlowing()) {
            controller.startSampling("Optimization Iter " + std::to_string(iter));
        }
        
        performOptimizationStep();
        
        // Update performance metrics for auto-sampling
        double util = measureGPUUtilization();
        double bw = measureMemoryBandwidth();
        double eff = measureComputeEfficiency();
        perfSampling.updateMetrics(util, bw, eff);
        
        if ((iter % 10 == 9) || hasConverged()) {
            controller.stopSampling();
        }
    }
    
    controller.generatePhaseReport();
}

class PhaseAnalyzer {
public:
    void analyzePhasePerformance(const std::vector<SamplingSession>& sessions) {
        std::cout << "Phase Performance Analysis:" << std::endl;
        
        for (const auto& session : sessions) {
            double duration = session.endTime - session.startTime;
            double sampleRate = session.samplesCollected / duration;
            
            std::cout << "Phase: " << session.phaseName << std::endl;
            std::cout << "  Duration: " << duration << "ms" << std::endl;
            std::cout << "  Samples: " << session.samplesCollected << std::endl;
            std::cout << "  Sample Rate: " << sampleRate << " samples/ms" << std::endl;
            
            // Analyze sample distribution
            analyzeSampleDistribution(session);
            
            // Identify hotspots
            identifyPhaseHotspots(session);
        }
    }
    
private:
    void analyzeSampleDistribution(const SamplingSession& session) {
        // Analyze how samples are distributed across functions/modules
        auto samples = getPCSamplesForSession(session);
        std::map<std::string, uint32_t> functionCounts;
        
        for (const auto& sample : samples) {
            std::string functionName = getFunctionName(sample.pc);
            functionCounts[functionName]++;
        }
        
        std::cout << "  Top Functions:" << std::endl;
        auto sortedFunctions = sortByCount(functionCounts);
        for (int i = 0; i < std::min(5, (int)sortedFunctions.size()); i++) {
            const auto& [funcName, count] = sortedFunctions[i];
            double percentage = (double)count / session.samplesCollected * 100;
            std::cout << "    " << funcName << ": " << percentage << "%" << std::endl;
        }
    }
};