@ -4,6 +4,8 @@
* Supports git - upload - pack protocol for fetching commits , trees , and blobs * Supports git - upload - pack protocol for fetching commits , trees , and blobs
* / * /
import { inflate , inflateRaw } from 'pako' ;
export interface GitObject { export interface GitObject {
type : 'commit' | 'tree' | 'blob' | 'tag' ; type : 'commit' | 'tree' | 'blob' | 'tag' ;
sha : string ; sha : string ;
@ -43,13 +45,27 @@ export async function fetchGitObjects(
const cleanUrl = repoUrl . endsWith ( '.git' ) ? repoUrl : ` ${ repoUrl } .git ` ; const cleanUrl = repoUrl . endsWith ( '.git' ) ? repoUrl : ` ${ repoUrl } .git ` ;
const uploadPackUrl = ` ${ cleanUrl } /git-upload-pack ` ; const uploadPackUrl = ` ${ cleanUrl } /git-upload-pack ` ;
// Build git-upload-pack request
// Build git-upload-pack request in pkt-line format
// Format: "want <sha>\n" for each object, then "done\n"
// Pkt-line format: 4 hex chars (length including 4-char prefix) + data + newline
// Example: "0032want 72a544a5b98007d3ef640e020dd70fe2ed129bb0\n"
// The length includes the 4-char prefix itself
function encodePktLine ( line : string ) : string {
const lineWithNewline = line + '\n' ;
const totalLength = 4 + lineWithNewline . length ;
const lengthHex = totalLength . toString ( 16 ) . padStart ( 4 , '0' ) ;
return lengthHex + lineWithNewline ;
}
let requestBody = '' ; let requestBody = '' ;
// Send capabilities (optional but recommended)
const capabilities = 'multi_ack thin-pack side-band side-band-64k ofs-delta shallow deepen-since deepen-not deepen-relative no-progress include-tag' ;
for ( const sha of wantShas ) { for ( const sha of wantShas ) {
requestBody += ` want ${ sha } \ n ` ; requestBody += encodePktLine ( ` want ${ sha } ${ capabilities } ` ) ;
} }
requestBody += 'done\n' ; // Send "done" to indicate we're done with wants
requestBody += encodePktLine ( 'done' ) ;
// Final "0000" to end the request
requestBody += '0000' ;
// Use proxy to avoid CORS
// Use proxy to avoid CORS
const proxyUrl = ` /api/gitea-proxy/git-upload-pack?url= ${ encodeURIComponent ( uploadPackUrl ) } ` ; const proxyUrl = ` /api/gitea-proxy/git-upload-pack?url= ${ encodeURIComponent ( uploadPackUrl ) } ` ;
@ -68,16 +84,322 @@ export async function fetchGitObjects(
return objects ; return objects ;
} }
// Parse packfile response
// Parse git-upload-pack response
// This is simplified - full packfile parsing is complex
// The response is in pkt-line format initially, then the packfile
// Format: pkt-line messages (like "NAK\n" or "ACK <sha>\n"), then packfile
const arrayBuffer = await response . arrayBuffer ( ) ; const arrayBuffer = await response . arrayBuffer ( ) ;
const data = new Uint8Array ( arrayBuffer ) ; const data = new Uint8Array ( arrayBuffer ) ;
// Basic packfile parsing (simplified)
console . log ( ` [Git Protocol] Received ${ data . length } bytes from git-upload-pack ` ) ;
// Real packfiles have a complex format with deltas, etc.
// For now, we'll use a simpler approach: fetch objects individually via HTTP
return objects ; if ( data . length === 0 ) {
console . warn ( '[Git Protocol] Empty response from git-upload-pack' ) ;
return objects ;
}
// Log first 200 bytes for debugging
const firstBytes = Array . from ( data . slice ( 0 , Math . min ( 200 , data . length ) ) )
. map ( b = > b . toString ( 16 ) . padStart ( 2 , '0' ) ) . join ( ' ' ) ;
console . log ( ` [Git Protocol] First 200 bytes (hex): ${ firstBytes } ` ) ;
// Find the packfile start (look for "PACK" header)
// Skip all pkt-line format messages first
const decoder = new TextDecoder ( ) ;
let packfileStart = - 1 ;
for ( let i = 0 ; i < data . length - 4 ; i ++ ) {
try {
const header = decoder . decode ( data . slice ( i , i + 4 ) ) ;
if ( header === 'PACK' ) {
packfileStart = i ;
break ;
}
} catch {
// Invalid UTF-8, continue
}
}
// Check if it starts with PACK already
try {
const firstHeader = decoder . decode ( data . slice ( 0 , 4 ) ) ;
if ( firstHeader === 'PACK' ) {
packfileStart = 0 ;
console . log ( '[Git Protocol] Packfile starts at beginning of response' ) ;
}
} catch {
// Not UTF-8, continue
}
// If not found at start, search for PACK header
if ( packfileStart === - 1 ) {
// Try to parse pkt-line format to find packfile
// Pkt-line format: 4 hex chars (length) + data
let pos = 0 ;
while ( pos < data . length - 4 ) {
try {
const lengthHex = decoder . decode ( data . slice ( pos , pos + 4 ) ) ;
const length = parseInt ( lengthHex , 16 ) ;
if ( isNaN ( length ) || length < 0 || length > 65535 ) {
// Not a valid pkt-line, might be packfile start
try {
const header = decoder . decode ( data . slice ( pos , pos + 4 ) ) ;
if ( header === 'PACK' ) {
packfileStart = pos ;
console . log ( ` [Git Protocol] Found packfile at offset ${ pos } after invalid pkt-line ` ) ;
break ;
}
} catch {
// Not UTF-8, continue
}
pos ++ ;
continue ;
}
if ( length === 0 ) {
// End marker (0000), packfile should follow
pos += 4 ;
if ( pos < data . length - 4 ) {
try {
const header = decoder . decode ( data . slice ( pos , pos + 4 ) ) ;
if ( header === 'PACK' ) {
packfileStart = pos ;
console . log ( ` [Git Protocol] Found packfile at offset ${ pos } after end marker ` ) ;
break ;
}
} catch {
// Not UTF-8, continue
}
}
pos ++ ;
continue ;
}
// Valid pkt-line, skip it
pos += length ;
} catch {
// Invalid UTF-8, try next position
pos ++ ;
}
}
}
// If still not found, try binary search for "PACK"
if ( packfileStart === - 1 ) {
for ( let i = 0 ; i < data . length - 4 ; i ++ ) {
if ( data [ i ] === 0x50 && data [ i + 1 ] === 0x41 && data [ i + 2 ] === 0x43 && data [ i + 3 ] === 0x4B ) {
// "PACK" in ASCII
packfileStart = i ;
console . log ( ` [Git Protocol] Found packfile at offset ${ i } via binary search ` ) ;
break ;
}
}
}
if ( packfileStart === - 1 ) {
// Try to decode as text to see what we got
try {
const textResponse = decoder . decode ( data . slice ( 0 , Math . min ( 500 , data . length ) ) ) ;
console . warn ( '[Git Protocol] No packfile found in response. Response text:' , textResponse ) ;
} catch {
console . warn ( '[Git Protocol] No packfile found in response. Response is binary, first 100 bytes (hex):' ,
Array . from ( data . slice ( 0 , 100 ) ) . map ( ( b : number ) = > b . toString ( 16 ) . padStart ( 2 , '0' ) ) . join ( ' ' ) ) ;
}
return objects ;
}
// Extract packfile data
const packfileData = data . slice ( packfileStart ) ;
if ( packfileData . length < 12 ) {
console . warn ( ` [Git Protocol] Packfile too small: ${ packfileData . length } bytes (need at least 12) ` ) ;
return objects ;
}
// Packfile format:
// Header: "PACK" (4 bytes) + version (4 bytes) + object count (4 bytes)
// Then objects in packfile format
// Footer: SHA1 checksum (20 bytes)
// Check packfile header
const header = new TextDecoder ( ) . decode ( packfileData . slice ( 0 , 4 ) ) ;
if ( header !== 'PACK' ) {
console . warn ( '[Git Protocol] Invalid packfile header:' , header ) ;
return objects ;
}
// Read version and object count (big-endian)
const view = new DataView ( packfileData . buffer , packfileData . byteOffset ) ;
const version = view . getUint32 ( 4 , false ) ;
const objectCount = view . getUint32 ( 8 , false ) ;
console . log ( ` [Git Protocol] Packfile found at offset ${ packfileStart } , version: ${ version } , objects: ${ objectCount } ` ) ;
// Parse objects (simplified - only handles non-delta objects)
let pos = 12 ;
for ( let i = 0 ; i < objectCount && pos < packfileData . length - 20 ; i ++ ) {
try {
// Read object type and size (variable-length encoding)
let byte = packfileData [ pos ++ ] ;
const type = ( byte >> 4 ) & 0x07 ; // 3 bits for type
let size = byte & 0x0F ; // 4 bits for size start
let shift = 4 ;
// Continue reading size if MSB is set
while ( byte & 0x80 ) {
if ( pos >= packfileData . length ) break ;
byte = packfileData [ pos ++ ] ;
size |= ( byte & 0x7F ) << shift ;
shift += 7 ;
}
// Packfile objects are zlib-compressed (raw deflate, no headers)
// We need to read the compressed data and decompress it
// The compressed size is variable, so we'll read incrementally
let objectData : Uint8Array | null = null ;
let objectType : GitObject [ 'type' ] ;
const compressedStart = pos ;
let compressedEnd = pos ;
// Try to decompress chunks of data until we get the expected size
// Start with a reasonable estimate (compressed is usually 50-80% of uncompressed)
// We'll try progressively larger chunks until decompression succeeds with the right size
let compressedSize = Math . max ( 32 , Math . ceil ( size * 0.5 ) ) ; // Start with 50% estimate, minimum 32 bytes
let attempts = 0 ;
const maxAttempts = 10 ; // Allow more attempts for finding the right compressed size
while ( attempts < maxAttempts && pos + compressedSize <= packfileData . length ) {
try {
const compressedChunk = packfileData . slice ( pos , pos + compressedSize ) ;
// Try to decompress using pako (raw deflate)
const decompressed = inflateRaw ( compressedChunk ) ;
if ( decompressed . length === size ) {
// Perfect match!
objectData = decompressed ;
compressedEnd = pos + compressedSize ;
break ;
} else if ( decompressed . length > size ) {
// Got more than expected - this means we read too much compressed data
// Try a smaller chunk
compressedSize = Math . max ( 32 , Math . floor ( compressedSize * 0.8 ) ) ;
attempts ++ ;
continue ;
} else if ( decompressed . length < size ) {
// Need more compressed data - the stream wasn't complete
compressedSize = Math . min ( Math . ceil ( compressedSize * 1.3 ) , data . length - pos ) ;
attempts ++ ;
continue ;
}
} catch ( decompressError ) {
// Decompression failed - might need more data (incomplete stream) or wrong starting position
if ( attempts < 3 ) {
// Early attempts - try reading more data (incomplete stream)
compressedSize = Math . min ( Math . ceil ( compressedSize * 1.5 ) , packfileData . length - pos ) ;
attempts ++ ;
continue ;
} else {
// Multiple failures - might be wrong format or corrupted
console . warn ( ` [Git Protocol] Failed to decompress object ${ i } after ${ attempts } attempts: ` , decompressError ) ;
// Skip this object - approximate position (assume ~60% compression)
pos += Math . min ( Math . ceil ( size * 0.6 ) , packfileData . length - pos ) ;
break ;
}
}
}
if ( ! objectData ) {
console . warn ( ` [Git Protocol] Could not decompress object ${ i } , skipping ` ) ;
// Skip this object - approximate position
pos += Math . min ( size / 2 , packfileData . length - pos ) ;
continue ;
}
pos = compressedEnd ; // Update position after reading compressed data
if ( type === 1 ) { // commit
objectType = 'commit' ;
} else if ( type === 2 ) { // tree
objectType = 'tree' ;
} else if ( type === 3 ) { // blob
objectType = 'blob' ;
} else if ( type === 4 ) { // tag
objectType = 'tag' ;
} else if ( type === 6 || type === 7 ) { // OFS_DELTA or REF_DELTA - skip for now
console . warn ( ` [Git Protocol] Delta object at index ${ i } , skipping (not implemented) ` ) ;
// Skip delta base reference and data
if ( type === 7 ) { // REF_DELTA has 20-byte base SHA
pos += 20 ;
} else { // OFS_DELTA has variable-length offset
let offsetByte = packfileData [ pos ++ ] ;
let offset = offsetByte & 0x7F ;
let shift = 7 ;
while ( offsetByte & 0x80 ) {
offsetByte = packfileData [ pos ++ ] ;
offset = ( ( offset + 1 ) << 7 ) | ( offsetByte & 0x7F ) ;
}
}
// Skip delta data (compressed) - read size first
let deltaSize = 0 ;
let deltaByte = packfileData [ pos ++ ] ;
deltaSize = deltaByte & 0x7F ;
let deltaShift = 7 ;
while ( deltaByte & 0x80 ) {
deltaByte = packfileData [ pos ++ ] ;
deltaSize |= ( deltaByte & 0x7F ) << deltaShift ;
deltaShift += 7 ;
}
// Skip the actual compressed delta data (approximate)
pos += Math . min ( deltaSize * 2 , packfileData . length - pos ) ;
continue ;
} else {
console . warn ( ` [Git Protocol] Unknown object type ${ type } at index ${ i } ` ) ;
continue ;
}
pos = compressedEnd ; // Update position
// Store object temporarily (we'll match by SHA after hashing)
const tempKey = ` obj_ ${ i } ` ;
objects . set ( tempKey , {
type : objectType ,
sha : tempKey , // Temporary, will be replaced after hashing
size : objectData.length ,
data : objectData
} ) ;
} catch ( error ) {
console . warn ( ` [Git Protocol] Error parsing object ${ i } : ` , error ) ;
break ;
}
}
// Match objects to requested SHAs by hashing
// This is a simplified approach - we hash each object and match
const matchedObjects = new Map < string , GitObject > ( ) ;
for ( const [ tempSha , obj ] of objects . entries ( ) ) {
// Calculate actual SHA1 using Web Crypto API
try {
const header = ` ${ obj . type } ${ obj . size } \ 0 ` ;
const headerBytes = new TextEncoder ( ) . encode ( header ) ;
const combined = new Uint8Array ( headerBytes . length + obj . data . length ) ;
combined . set ( headerBytes , 0 ) ;
combined . set ( obj . data , headerBytes . length ) ;
const hashBuffer = await crypto . subtle . digest ( 'SHA-1' , combined ) ;
const hashArray = Array . from ( new Uint8Array ( hashBuffer ) ) ;
const actualSha = hashArray . map ( b = > b . toString ( 16 ) . padStart ( 2 , '0' ) ) . join ( '' ) ;
if ( wantShas . includes ( actualSha ) ) {
matchedObjects . set ( actualSha , { . . . obj , sha : actualSha } ) ;
}
} catch ( error ) {
console . warn ( ` [Git Protocol] Error hashing object: ` , error ) ;
}
}
console . log ( ` [Git Protocol] Matched ${ matchedObjects . size } of ${ wantShas . length } requested objects ` ) ;
return matchedObjects ;
} catch ( error ) { } catch ( error ) {
console . error ( 'Error fetching git objects:' , error ) ; console . error ( 'Error fetching git objects:' , error ) ;
return objects ; return objects ;
@ -86,7 +408,8 @@ export async function fetchGitObjects(
/ * * / * *
* Fetch a single git object by SHA * Fetch a single git object by SHA
* Uses HTTP endpoint with decompression via proxy * Uses GRASP / git - natural - api HTTP endpoint : /objects/ { prefix } / { suffix }
* Handles zlib decompression client - side ( raw deflate format )
* / * /
export async function fetchGitObject ( export async function fetchGitObject (
repoUrl : string , repoUrl : string ,
@ -95,56 +418,150 @@ export async function fetchGitObject(
try { try {
const cleanUrl = repoUrl . endsWith ( '.git' ) ? repoUrl : ` ${ repoUrl } .git ` ; const cleanUrl = repoUrl . endsWith ( '.git' ) ? repoUrl : ` ${ repoUrl } .git ` ;
// Try HTTP endpoint with decompression (proxy handles zlib decompression)
// GRASP/git-natural-api uses simple HTTP endpoint: /objects/{prefix}/{suffix}
const objectUrl = ` ${ cleanUrl } /objects/ ${ sha . substring ( 0 , 2 ) } / ${ sha . substring ( 2 ) } ` ; const objectUrl = ` ${ cleanUrl } /objects/ ${ sha . substring ( 0 , 2 ) } / ${ sha . substring ( 2 ) } ` ;
try { try {
// Always use proxy for GRASP URLs to avoid CORS issues
// The proxy returns raw compressed data that we decompress client-side
const proxyUrl = ` /api/gitea-proxy/git-object?url= ${ encodeURIComponent ( objectUrl ) } ` ; const proxyUrl = ` /api/gitea-proxy/git-object?url= ${ encodeURIComponent ( objectUrl ) } ` ;
const response = await fetch ( proxyUrl ) ; const response = await fetch ( proxyUrl ) ;
if ( response . ok ) {
const data = new Uint8Array ( await response . arrayBuffer ( ) ) ; if ( ! response . ok ) {
// Parse git object format (already decompressed by proxy)
if ( response . status === 404 ) {
return parseGitObject ( data , sha ) ; console . warn ( ` [Git Protocol] Object not found: ${ sha } (404) ` ) ;
return null ;
}
throw new Error ( ` HTTP ${ response . status } : ${ response . statusText } ` ) ;
}
const data = new Uint8Array ( await response . arrayBuffer ( ) ) ;
// Git objects can be:
// 1. Zlib-compressed (with zlib headers) - when served via HTTP /objects/{prefix}/{suffix}
// 2. Raw deflate (no headers) - when in packfiles
// 3. Already decompressed - some servers serve them this way
//
// Strategy: Check format and decompress accordingly
// Check if data starts with zlib header (0x78 0x01, 0x78 0x9C, 0x78 0xDA, etc.)
const hasZlibHeader = data . length >= 2 && data [ 0 ] === 0x78 &&
( data [ 1 ] === 0x01 || data [ 1 ] === 0x9C || data [ 1 ] === 0xDA || data [ 1 ] === 0x5E ) ;
// First, try parsing as already decompressed (fastest check)
const directParse = parseGitObject ( data , sha ) ;
if ( directParse ) {
console . log ( ` [Git Protocol] Object ${ sha } was already decompressed ` ) ;
return directParse ;
}
// If direct parse failed and data looks compressed, try decompression
let decompressed : Uint8Array ;
try {
if ( hasZlibHeader ) {
// Zlib-compressed (with headers) - use inflate
decompressed = inflate ( data ) ;
console . log ( ` [Git Protocol] Successfully decompressed object ${ sha } (zlib, ${ data . length } -> ${ decompressed . length } bytes) ` ) ;
} else {
// Raw deflate (no headers) - use inflateRaw
decompressed = inflateRaw ( data ) ;
console . log ( ` [Git Protocol] Successfully decompressed object ${ sha } (raw deflate, ${ data . length } -> ${ decompressed . length } bytes) ` ) ;
}
} catch ( decompressError ) {
// Decompression failed - log for debugging
console . warn ( ` [Git Protocol] Decompression failed for ${ sha } : ` , decompressError ) ;
console . warn ( ` [Git Protocol] Data length: ${ data . length } , first 20 bytes (hex): ` ,
Array . from ( data . slice ( 0 , 20 ) ) . map ( b = > b . toString ( 16 ) . padStart ( 2 , '0' ) ) . join ( ' ' ) ) ;
console . warn ( ` [Git Protocol] Has zlib header: ${ hasZlibHeader } ` ) ;
return null ;
} }
// Parse decompressed git object format: "<type> <size>\0<data>"
return parseGitObject ( decompressed , sha ) ;
} catch ( error ) { } catch ( error ) {
console . warn ( ` Failed to fetch git object via HTTP: ${ error } ` ) ; console . warn ( ` [Git Protocol] Failed to fetch git object via HTTP: ${ error } ` ) ;
} }
// Fallback: use git-upload-pack for single object
// Fallback: use git-upload-pack for single object (only if HTTP fails)
console . log ( ` [Git Protocol] Falling back to git-upload-pack for ${ sha } ` ) ;
const objects = await fetchGitObjects ( repoUrl , [ sha ] ) ; const objects = await fetchGitObjects ( repoUrl , [ sha ] ) ;
return objects . get ( sha ) || null ; return objects . get ( sha ) || null ;
} catch ( error ) { } catch ( error ) {
console . error ( ` Error fetching git object ${ sha } : ` , error ) ; console . error ( ` [Git Protocol] Error fetching git object ${ sha } : ` , error ) ;
return null ; return null ;
} }
} }
/ * * / * *
* Parse git object from raw data ( already decompressed by proxy ) * Parse git object from raw data ( decompressed )
* Git objects have header : "<type> <size>\0<data>" * Git objects have header : "<type> <size>\0<data>"
* The null byte ( \ 0 ) separates the header from the content
* / * /
function parseGitObject ( data : Uint8Array , sha : string ) : GitObject | null { function parseGitObject ( data : Uint8Array , sha : string ) : GitObject | null {
try { try {
// Data is already decompressed by proxy
if ( data . length === 0 ) {
// Format: "commit 1234\0<data>" or "tree 5678\0<data>" etc.
console . warn ( ` [Git Protocol] Empty data for object ${ sha } ` ) ;
const text = new TextDecoder ( ) . decode ( data ) ; return null ;
const headerMatch = text . match ( /^(\w+) (\d+)\0/ ) ; }
// Find the null byte that separates header from content
let nullByteIndex = - 1 ;
for ( let i = 0 ; i < Math . min ( 100 , data . length ) ; i ++ ) {
if ( data [ i ] === 0 ) {
nullByteIndex = i ;
break ;
}
}
if ( nullByteIndex === - 1 ) {
// No null byte found - this is not a valid git object format
// Log first bytes for debugging
const firstBytes = Array . from ( data . slice ( 0 , Math . min ( 50 , data . length ) ) )
. map ( b = > b . toString ( 16 ) . padStart ( 2 , '0' ) ) . join ( ' ' ) ;
const firstText = new TextDecoder ( 'utf-8' , { fatal : false } ) . decode ( data . slice ( 0 , Math . min ( 50 , data . length ) ) ) ;
console . warn ( ` [Git Protocol] No null byte found in object ${ sha } ` ) ;
console . warn ( ` [Git Protocol] First 50 bytes (hex): ${ firstBytes } ` ) ;
console . warn ( ` [Git Protocol] First 50 bytes (text): ${ firstText } ` ) ;
return null ;
}
// Parse header: "<type> <size>\0"
const headerBytes = data . slice ( 0 , nullByteIndex ) ;
const headerText = new TextDecoder ( 'utf-8' , { fatal : false } ) . decode ( headerBytes ) ;
const headerMatch = headerText . match ( /^(\w+) (\d+)$/ ) ;
if ( ! headerMatch ) { if ( ! headerMatch ) {
console . warn ( ` [Git Protocol] Invalid header format for ${ sha } : " ${ headerText } " ` ) ;
console . warn ( ` [Git Protocol] Header bytes (hex): ` ,
Array . from ( headerBytes ) . map ( b = > b . toString ( 16 ) . padStart ( 2 , '0' ) ) . join ( ' ' ) ) ;
return null ; return null ;
} }
const type = headerMatch [ 1 ] as GitObject [ 'type' ] ; const type = headerMatch [ 1 ] as GitObject [ 'type' ] ;
const size = parseInt ( headerMatch [ 2 ] , 10 ) ; const size = parseInt ( headerMatch [ 2 ] , 10 ) ;
const contentStart = headerMatch [ 0 ] . length ;
if ( isNaN ( size ) || size < 0 ) {
console . warn ( ` [Git Protocol] Invalid size in header for ${ sha } : " ${ headerMatch [ 2 ] } " ` ) ;
return null ;
}
const contentStart = nullByteIndex + 1 ; // Skip the null byte
const content = data . slice ( contentStart ) ; const content = data . slice ( contentStart ) ;
// Verify we have enough data
if ( content . length < size ) {
console . warn ( ` [Git Protocol] Content size mismatch for ${ sha } : expected ${ size } , got ${ content . length } ` ) ;
// Still return what we have, but log the issue
}
return { return {
type , type ,
sha , sha ,
size , size ,
data : content data : content.slice ( 0 , size ) // Take only the expected size
} ; } ;
} catch ( error ) { } catch ( error ) {
console . error ( 'Error parsing git object:' , error ) ; console . error ( ` [Git Protocol] Error parsing git object ${ sha } : ` , error ) ;
return null ; return null ;
} }
} }
